diff options
Diffstat (limited to 'src/bootstrap')
39 files changed, 1993 insertions, 872 deletions
diff --git a/src/bootstrap/CHANGELOG.md b/src/bootstrap/CHANGELOG.md index 4105fa5ec..74dd22df9 100644 --- a/src/bootstrap/CHANGELOG.md +++ b/src/bootstrap/CHANGELOG.md @@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `remote-test-server`'s `verbose` argument has been removed in favor of the `--verbose` flag - `remote-test-server`'s `remote` argument has been removed in favor of the `--bind` flag. Use `--bind 0.0.0.0:12345` to replicate the behavior of the `remote` argument. - `x.py fmt` now formats only files modified between the merge-base of HEAD and the last commit in the master branch of the rust-lang repository and the current working directory. To restore old behaviour, use `x.py fmt .`. The check mode is not affected by this change. [#105702](https://github.com/rust-lang/rust/pull/105702) +- The `llvm.version-check` config option has been removed. Older versions were never supported. If you still need to support older versions (e.g. you are applying custom patches), patch `check_llvm_version` in bootstrap to change the minimum version. [#108619](https://github.com/rust-lang/rust/pull/108619) +- The `rust.ignore-git` option has been renamed to `rust.omit-git-hash`. [#110059](https://github.com/rust-lang/rust/pull/110059) ### Non-breaking changes @@ -24,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - If you have Rust already installed, `x.py` will now infer the host target from the default rust toolchain. [#78513](https://github.com/rust-lang/rust/pull/78513) - Add options for enabling overflow checks, one for std (`overflow-checks-std`) and one for everything else (`overflow-checks`). Both default to false. +- Add llvm option `enable-warnings` to have control on llvm compilation warnings. Default to false. ## [Version 2] - 2020-09-25 @@ -46,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Add `--keep-stage-std`, which behaves like `keep-stage` but allows the stage 0 compiler artifacts (i.e., stage1/bin/rustc) to be rebuilt if changed [#77120](https://github.com/rust-lang/rust/pull/77120). +- File locking is now used to avoid collisions between multiple running instances of `x.py` (e.g. when using `rust-analyzer` and `x.py` at the same time). Note that Solaris and possibly other non Unix and non Windows systems don't support it [#108607](https://github.com/rust-lang/rust/pull/108607). This might possibly lead to build data corruption. ## [Version 1] - 2020-09-11 diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index e861d520c..a158d1f71 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -44,6 +44,8 @@ dependencies = [ "getopts", "hex", "ignore", + "is-terminal", + "junction", "libc", "object", "once_cell", @@ -55,9 +57,10 @@ dependencies = [ "sha2", "sysinfo", "tar", + "termcolor", "toml", "walkdir", - "winapi", + "windows", "xz2", ] @@ -199,13 +202,13 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "errno" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys", ] [[package]] @@ -220,9 +223,9 @@ dependencies = [ [[package]] name = "fd-lock" -version = "3.0.8" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb21c69b9fea5e15dbc1049e4b77145dd0ba1c84019c488102de0dc4ea4b0a27" +checksum = "9799aefb4a2e4a01cc47610b1dd47c18ab13d991f27bbcaed9296f5a53d5cbad" dependencies = [ "cfg-if", "rustix", @@ -289,6 +292,12 @@ dependencies = [ ] [[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -314,21 +323,44 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.1" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d367024b3f3414d8e01f437f704f41a9f64ab36f9067fa73e526ad4c763c87" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ + "hermit-abi 0.3.1", "libc", "windows-sys", ] [[package]] +name = "is-terminal" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys", +] + +[[package]] name = "itoa" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] +name = "junction" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca39ef0d69b18e6a2fd14c2f0a1d593200f4a4ed949b240b5917ab51fac754cb" +dependencies = [ + "scopeguard", + "winapi", +] + +[[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -336,15 +368,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "linux-raw-sys" -version = "0.1.3" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" [[package]] name = "log" @@ -396,7 +428,7 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", ] @@ -527,9 +559,9 @@ checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "rustix" -version = "0.36.3" +version = "0.37.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b1fbb4dfc4eb1d390c02df47760bb19a84bb80b301ecc947ab5406394d8223e" +checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849" dependencies = [ "bitflags", "errno", @@ -637,6 +669,15 @@ dependencies = [ ] [[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] name = "thread_local" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -721,10 +762,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + +[[package]] name = "windows-sys" -version = "0.42.0" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -737,45 +796,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "xattr" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 663987f11..eeda6d7c1 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -30,9 +30,9 @@ path = "bin/sccache-plus-cl.rs" test = false [dependencies] +is-terminal = "0.4" build_helper = { path = "../tools/build_helper" } cmake = "0.1.38" -fd-lock = "3.0.8" filetime = "0.2" getopts = "0.2.19" cc = "1.0.69" @@ -46,6 +46,7 @@ serde_derive = "1.0.137" serde_json = "1.0.2" sha2 = "0.10" tar = "0.4" +termcolor = "1.2.0" toml = "0.5" ignore = "0.4.10" opener = "0.5" @@ -56,18 +57,27 @@ walkdir = "2" # Dependencies needed by the build-metrics feature sysinfo = { version = "0.26.0", optional = true } -[target.'cfg(windows)'.dependencies.winapi] -version = "0.3" +# Solaris doesn't support flock() and thus fd-lock is not option now +[target.'cfg(not(target_os = "solaris"))'.dependencies] +fd-lock = "3.0.8" + +[target.'cfg(windows)'.dependencies.junction] +version = "1.0.0" + +[target.'cfg(windows)'.dependencies.windows] +version = "0.46.0" features = [ - "fileapi", - "ioapiset", - "jobapi2", - "handleapi", - "winioctl", - "psapi", - "impl-default", - "timezoneapi", - "winbase", + "Win32_Foundation", + "Win32_Security", + "Win32_Storage_FileSystem", + "Win32_System_Diagnostics_Debug", + "Win32_System_IO", + "Win32_System_Ioctl", + "Win32_System_JobObjects", + "Win32_System_ProcessStatus", + "Win32_System_SystemServices", + "Win32_System_Threading", + "Win32_System_Time", ] [dev-dependencies] diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 71eee8968..253d504d7 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -185,7 +185,7 @@ Some general areas that you may be interested in modifying are: If you make a major change, please remember to: + Update `VERSION` in `src/bootstrap/main.rs`. -* Update `changelog-seen = N` in `config.toml.example`. +* Update `changelog-seen = N` in `config.example.toml`. * Add an entry in `src/bootstrap/CHANGELOG.md`. A 'major change' includes diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs index 3856bb64f..912d875e4 100644 --- a/src/bootstrap/bin/main.rs +++ b/src/bootstrap/bin/main.rs @@ -7,15 +7,18 @@ use std::env; -use bootstrap::{t, Build, Config, Subcommand, VERSION}; +#[cfg(all(any(unix, windows), not(target_os = "solaris")))] +use bootstrap::t; +use bootstrap::{Build, Config, Subcommand, VERSION}; fn main() { let args = env::args().skip(1).collect::<Vec<_>>(); let config = Config::parse(&args); - let mut build_lock; - let _build_lock_guard; - if cfg!(any(unix, windows)) { + #[cfg(all(any(unix, windows), not(target_os = "solaris")))] + { + let mut build_lock; + let _build_lock_guard; let path = config.out.join("lock"); build_lock = fd_lock::RwLock::new(t!(std::fs::File::create(&path))); _build_lock_guard = match build_lock.try_write() { @@ -30,9 +33,9 @@ fn main() { t!(build_lock.write()) } }; - } else { - println!("warning: file locking not supported for target, not locking build directory"); } + #[cfg(any(not(any(unix, windows)), target_os = "solaris"))] + println!("warning: file locking not supported for target, not locking build directory"); // check_version warnings are not printed during setup let changelog_suggestion = @@ -44,8 +47,8 @@ fn main() { if suggest_setup { println!("warning: you have not made a `config.toml`"); println!( - "help: consider running `./x.py setup` or copying `config.toml.example` by running \ - `cp config.toml.example config.toml`" + "help: consider running `./x.py setup` or copying `config.example.toml` by running \ + `cp config.example.toml config.toml`" ); } else if let Some(suggestion) = &changelog_suggestion { println!("{}", suggestion); @@ -57,8 +60,8 @@ fn main() { if suggest_setup { println!("warning: you have not made a `config.toml`"); println!( - "help: consider running `./x.py setup` or copying `config.toml.example` by running \ - `cp config.toml.example config.toml`" + "help: consider running `./x.py setup` or copying `config.example.toml` by running \ + `cp config.example.toml config.toml`" ); } else if let Some(suggestion) = &changelog_suggestion { println!("{}", suggestion); @@ -125,7 +128,7 @@ fn get_lock_owner(f: &std::path::Path) -> Option<u64> { }) } -#[cfg(not(target_os = "linux"))] +#[cfg(not(any(target_os = "linux", target_os = "solaris")))] fn get_lock_owner(_: &std::path::Path) -> Option<u64> { // FIXME: Implement on other OS's None diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index 9611c866d..040fec361 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -281,41 +281,49 @@ fn format_rusage_data(_child: Child) -> Option<String> { #[cfg(windows)] fn format_rusage_data(child: Child) -> Option<String> { use std::os::windows::io::AsRawHandle; - use winapi::um::{processthreadsapi, psapi, timezoneapi}; - let handle = child.as_raw_handle(); - macro_rules! try_bool { - ($e:expr) => { - if $e != 1 { - return None; - } - }; - } + + use windows::{ + Win32::Foundation::HANDLE, + Win32::System::ProcessStatus::{ + K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS, PROCESS_MEMORY_COUNTERS_EX, + }, + Win32::System::Threading::GetProcessTimes, + Win32::System::Time::FileTimeToSystemTime, + }; + + let handle = HANDLE(child.as_raw_handle() as isize); let mut user_filetime = Default::default(); let mut user_time = Default::default(); let mut kernel_filetime = Default::default(); let mut kernel_time = Default::default(); - let mut memory_counters = psapi::PROCESS_MEMORY_COUNTERS::default(); + let mut memory_counters = PROCESS_MEMORY_COUNTERS::default(); unsafe { - try_bool!(processthreadsapi::GetProcessTimes( + GetProcessTimes( handle, &mut Default::default(), &mut Default::default(), &mut kernel_filetime, &mut user_filetime, - )); - try_bool!(timezoneapi::FileTimeToSystemTime(&user_filetime, &mut user_time)); - try_bool!(timezoneapi::FileTimeToSystemTime(&kernel_filetime, &mut kernel_time)); - - // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process - // with the given handle and none of that process's children. - try_bool!(psapi::GetProcessMemoryInfo( - handle as _, - &mut memory_counters as *mut _ as _, - std::mem::size_of::<psapi::PROCESS_MEMORY_COUNTERS_EX>() as u32, - )); + ) + } + .ok() + .ok()?; + unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok().ok()?; + unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok().ok()?; + + // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process + // with the given handle and none of that process's children. + unsafe { + K32GetProcessMemoryInfo( + handle, + &mut memory_counters, + std::mem::size_of::<PROCESS_MEMORY_COUNTERS_EX>() as u32, + ) } + .ok() + .ok()?; // Guide on interpreting these numbers: // https://docs.microsoft.com/en-us/windows/win32/psapi/process-memory-usage-information diff --git a/src/bootstrap/bolt.rs b/src/bootstrap/bolt.rs index ea37cd470..10e6d2e7d 100644 --- a/src/bootstrap/bolt.rs +++ b/src/bootstrap/bolt.rs @@ -1,46 +1,40 @@ use std::path::Path; use std::process::Command; -/// Uses the `llvm-bolt` binary to instrument the binary/library at the given `path` with BOLT. +/// Uses the `llvm-bolt` binary to instrument the artifact at the given `path` with BOLT. /// When the instrumented artifact is executed, it will generate BOLT profiles into /// `/tmp/prof.fdata.<pid>.fdata`. -pub fn instrument_with_bolt_inplace(path: &Path) { - let dir = std::env::temp_dir(); - let instrumented_path = dir.join("instrumented.so"); - +/// Creates the instrumented artifact at `output_path`. +pub fn instrument_with_bolt(path: &Path, output_path: &Path) { let status = Command::new("llvm-bolt") .arg("-instrument") .arg(&path) // Make sure that each process will write its profiles into a separate file .arg("--instrumentation-file-append-pid") .arg("-o") - .arg(&instrumented_path) + .arg(output_path) .status() .expect("Could not instrument artifact using BOLT"); if !status.success() { panic!("Could not instrument {} with BOLT, exit code {:?}", path.display(), status.code()); } - - std::fs::copy(&instrumented_path, path).expect("Cannot copy instrumented artifact"); - std::fs::remove_file(instrumented_path).expect("Cannot delete instrumented artifact"); } -/// Uses the `llvm-bolt` binary to optimize the binary/library at the given `path` with BOLT, +/// Uses the `llvm-bolt` binary to optimize the artifact at the given `path` with BOLT, /// using merged profiles from `profile_path`. /// /// The recorded profiles have to be merged using the `merge-fdata` tool from LLVM and the merged /// profile path should be then passed to this function. -pub fn optimize_library_with_bolt_inplace(path: &Path, profile_path: &Path) { - let dir = std::env::temp_dir(); - let optimized_path = dir.join("optimized.so"); - +/// +/// Creates the optimized artifact at `output_path`. +pub fn optimize_with_bolt(path: &Path, profile_path: &Path, output_path: &Path) { let status = Command::new("llvm-bolt") .arg(&path) .arg("-data") .arg(&profile_path) .arg("-o") - .arg(&optimized_path) + .arg(output_path) // Reorder basic blocks within functions .arg("-reorder-blocks=ext-tsp") // Reorder functions within the binary @@ -51,8 +45,6 @@ pub fn optimize_library_with_bolt_inplace(path: &Path, profile_path: &Path) { .arg("-split-all-cold") // Move jump tables to a separate section .arg("-jump-tables=move") - // Use GNU_STACK program header for new segment (workaround for issues with strip/objcopy) - .arg("-use-gnu-stack") // Fold functions with identical code .arg("-icf=1") // Update DWARF debug info in the final binary @@ -65,7 +57,4 @@ pub fn optimize_library_with_bolt_inplace(path: &Path, profile_path: &Path) { if !status.success() { panic!("Could not optimize {} with BOLT, exit code {:?}", path.display(), status.code()); } - - std::fs::copy(&optimized_path, path).expect("Cannot copy optimized artifact"); - std::fs::remove_file(optimized_path).expect("Cannot delete optimized artifact"); } diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 013d1ab52..025145244 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -304,6 +304,7 @@ def default_build_triple(verbose): 'i486': 'i686', 'i686': 'i686', 'i786': 'i686', + 'loongarch64': 'loongarch64', 'm68k': 'm68k', 'powerpc': 'powerpc', 'powerpc64': 'powerpc64', @@ -741,6 +742,9 @@ class RustBuild(object): env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ (os.pathsep + env["LIBRARY_PATH"]) \ if "LIBRARY_PATH" in env else "" + env["LIBPATH"] = os.path.join(self.bin_root(), "lib") + \ + (os.pathsep + env["LIBPATH"]) \ + if "LIBPATH" in env else "" # Export Stage0 snapshot compiler related env variables build_section = "target.{}".format(self.build) diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py index 06ca3ce21..20bd71f06 100644 --- a/src/bootstrap/bootstrap_test.py +++ b/src/bootstrap/bootstrap_test.py @@ -11,6 +11,7 @@ import sys from shutil import rmtree import bootstrap +import configure class VerifyTestCase(unittest.TestCase): @@ -74,12 +75,50 @@ class ProgramOutOfDate(unittest.TestCase): self.assertFalse(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) +class GenerateAndParseConfig(unittest.TestCase): + """Test that we can serialize and deserialize a config.toml file""" + def serialize_and_parse(self, args): + from io import StringIO + + section_order, sections, targets = configure.parse_args(args) + buffer = StringIO() + configure.write_config_toml(buffer, section_order, targets, sections) + build = bootstrap.RustBuild() + build.config_toml = buffer.getvalue() + + try: + import tomllib + # Verify this is actually valid TOML. + tomllib.loads(build.config_toml) + except ImportError: + print("warning: skipping TOML validation, need at least python 3.11", file=sys.stderr) + return build + + def test_no_args(self): + build = self.serialize_and_parse([]) + self.assertEqual(build.get_toml("changelog-seen"), '2') + self.assertIsNone(build.get_toml("llvm.download-ci-llvm")) + + def test_set_section(self): + build = self.serialize_and_parse(["--set", "llvm.download-ci-llvm"]) + self.assertEqual(build.get_toml("download-ci-llvm", section="llvm"), 'true') + + def test_set_target(self): + build = self.serialize_and_parse(["--set", "target.x86_64-unknown-linux-gnu.cc=gcc"]) + self.assertEqual(build.get_toml("cc", section="target.x86_64-unknown-linux-gnu"), 'gcc') + + # Uncomment when #108928 is fixed. + # def test_set_top_level(self): + # build = self.serialize_and_parse(["--set", "profile=compiler"]) + # self.assertEqual(build.get_toml("profile"), 'compiler') + if __name__ == '__main__': SUITE = unittest.TestSuite() TEST_LOADER = unittest.TestLoader() SUITE.addTest(doctest.DocTestSuite(bootstrap)) SUITE.addTests([ TEST_LOADER.loadTestsFromTestCase(VerifyTestCase), + TEST_LOADER.loadTestsFromTestCase(GenerateAndParseConfig), TEST_LOADER.loadTestsFromTestCase(ProgramOutOfDate)]) RUNNER = unittest.TextTestRunner(stream=sys.stdout, verbosity=2) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index b33fc02f4..e959ea06f 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -4,8 +4,9 @@ use std::collections::BTreeSet; use std::env; use std::ffi::OsStr; use std::fmt::{Debug, Write}; -use std::fs::{self}; +use std::fs::{self, File}; use std::hash::Hash; +use std::io::{BufRead, BufReader}; use std::ops::Deref; use std::path::{Component, Path, PathBuf}; use std::process::Command; @@ -16,7 +17,7 @@ use crate::config::{SplitDebuginfo, TargetSelection}; use crate::doc; use crate::flags::{Color, Subcommand}; use crate::install; -use crate::native; +use crate::llvm; use crate::run; use crate::setup; use crate::test; @@ -28,8 +29,11 @@ use crate::{clean, dist}; use crate::{Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; -// FIXME: replace with std::lazy after it gets stabilized and reaches beta -use once_cell::sync::Lazy; +// FIXME: +// - use std::lazy for `Lazy` +// - use std::cell for `OnceCell` +// Once they get stabilized and reach beta. +use once_cell::sync::{Lazy, OnceCell}; pub struct Builder<'a> { pub build: &'a Build, @@ -484,17 +488,43 @@ impl<'a> ShouldRun<'a> { // multiple aliases for the same job pub fn paths(mut self, paths: &[&str]) -> Self { + static SUBMODULES_PATHS: OnceCell<Vec<String>> = OnceCell::new(); + + let init_submodules_paths = |src: &PathBuf| { + let file = File::open(src.join(".gitmodules")).unwrap(); + + let mut submodules_paths = vec![]; + for line in BufReader::new(file).lines() { + if let Ok(line) = line { + let line = line.trim(); + + if line.starts_with("path") { + let actual_path = + line.split(' ').last().expect("Couldn't get value of path"); + submodules_paths.push(actual_path.to_owned()); + } + } + } + + submodules_paths + }; + + let submodules_paths = + SUBMODULES_PATHS.get_or_init(|| init_submodules_paths(&self.builder.src)); + self.paths.insert(PathSet::Set( paths .iter() .map(|p| { - // FIXME(#96188): make sure this is actually a path. - // This currently breaks for paths within submodules. - //assert!( - // self.builder.src.join(p).exists(), - // "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", - // p - //); + // assert only if `p` isn't submodule + if !submodules_paths.iter().find(|sm_p| p.contains(*sm_p)).is_some() { + assert!( + self.builder.src.join(p).exists(), + "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", + p + ); + } + TaskPath { path: p.into(), kind: Some(self.kind) } }) .collect(), @@ -561,6 +591,7 @@ pub enum Kind { Install, Run, Setup, + Suggest, } impl Kind { @@ -580,6 +611,7 @@ impl Kind { "install" => Kind::Install, "run" | "r" => Kind::Run, "setup" => Kind::Setup, + "suggest" => Kind::Suggest, _ => return None, }) } @@ -599,6 +631,7 @@ impl Kind { Kind::Install => "install", Kind::Run => "run", Kind::Setup => "setup", + Kind::Suggest => "suggest", } } } @@ -636,13 +669,13 @@ impl<'a> Builder<'a> { tool::Rustdoc, tool::Clippy, tool::CargoClippy, - native::Llvm, - native::Sanitizers, + llvm::Llvm, + llvm::Sanitizers, tool::Rustfmt, tool::Miri, tool::CargoMiri, - native::Lld, - native::CrtBeginEnd + llvm::Lld, + llvm::CrtBeginEnd ), Kind::Check | Kind::Clippy | Kind::Fix => describe!( check::Std, @@ -679,6 +712,7 @@ impl<'a> Builder<'a> { test::CrateRustdoc, test::CrateRustdocJsonTypes, test::CrateJsonDocLint, + test::SuggestTestsCrate, test::Linkcheck, test::TierCheck, test::ReplacePlaceholderTest, @@ -711,6 +745,7 @@ impl<'a> Builder<'a> { test::RustdocUi, test::RustdocJson, test::HtmlCheck, + test::RustInstaller, // Run bootstrap close to the end as it's unlikely to fail test::Bootstrap, // Run run-make last, since these won't pass without make on Windows @@ -740,6 +775,7 @@ impl<'a> Builder<'a> { doc::EmbeddedBook, doc::EditionGuide, doc::StyleGuide, + doc::Tidy, ), Kind::Dist => describe!( dist::Docs, @@ -795,7 +831,7 @@ impl<'a> Builder<'a> { Kind::Setup => describe!(setup::Profile, setup::Hook, setup::Link, setup::Vscode), Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std), // special-cased in Build::build() - Kind::Format => vec![], + Kind::Format | Kind::Suggest => vec![], } } @@ -859,6 +895,7 @@ impl<'a> Builder<'a> { Subcommand::Run { ref paths, .. } => (Kind::Run, &paths[..]), Subcommand::Clean { ref paths, .. } => (Kind::Clean, &paths[..]), Subcommand::Format { .. } => (Kind::Format, &[][..]), + Subcommand::Suggest { .. } => (Kind::Suggest, &[][..]), Subcommand::Setup { profile: ref path } => ( Kind::Setup, path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)), @@ -868,6 +905,21 @@ impl<'a> Builder<'a> { Self::new_internal(build, kind, paths.to_owned()) } + /// Creates a new standalone builder for use outside of the normal process + pub fn new_standalone( + build: &mut Build, + kind: Kind, + paths: Vec<PathBuf>, + stage: Option<u32>, + ) -> Builder<'_> { + // FIXME: don't mutate `build` + if let Some(stage) = stage { + build.config.stage = stage; + } + + Self::new_internal(build, kind, paths.to_owned()) + } + pub fn execute_cli(&self) { self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths); } @@ -910,14 +962,16 @@ impl<'a> Builder<'a> { /// new artifacts, it can't be used to rely on the presence of a particular /// sysroot. /// - /// See `force_use_stage1` for documentation on what each argument is. + /// See `force_use_stage1` and `force_use_stage2` for documentation on what each argument is. pub fn compiler_for( &self, stage: u32, host: TargetSelection, target: TargetSelection, ) -> Compiler { - if self.build.force_use_stage1(Compiler { stage, host }, target) { + if self.build.force_use_stage2(stage) { + self.compiler(2, self.config.build) + } else if self.build.force_use_stage1(stage, target) { self.compiler(1, self.config.build) } else { self.compiler(stage, host) @@ -1097,7 +1151,7 @@ impl<'a> Builder<'a> { /// check build or dry-run, where there's no need to build all of LLVM. fn llvm_config(&self, target: TargetSelection) -> Option<PathBuf> { if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() { - let native::LlvmResult { llvm_config, .. } = self.ensure(native::Llvm { target }); + let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target }); if llvm_config.is_file() { return Some(llvm_config); } @@ -1223,7 +1277,7 @@ impl<'a> Builder<'a> { // rustc_llvm. But if LLVM is stale, that'll be a tiny amount // of work comparatively, and we'd likely need to rebuild it anyway, // so that's okay. - if crate::native::prebuilt_llvm_config(self, target).is_err() { + if crate::llvm::prebuilt_llvm_config(self, target).is_err() { cargo.env("RUST_CHECK", "1"); } } @@ -1299,6 +1353,14 @@ impl<'a> Builder<'a> { } }; + // By default, windows-rs depends on a native library that doesn't get copied into the + // sysroot. Passing this cfg enables raw-dylib support instead, which makes the native + // library unnecessary. This can be removed when windows-rs enables raw-dylib + // unconditionally. + if let Mode::Rustc | Mode::ToolRustc = mode { + rustflags.arg("--cfg=windows_raw_dylib"); + } + if use_new_symbol_mangling { rustflags.arg("-Csymbol-mangling-version=v0"); } else { @@ -1718,6 +1780,15 @@ impl<'a> Builder<'a> { cargo.env("RUSTC_VERBOSE", self.verbosity.to_string()); + // Downstream forks of the Rust compiler might want to use a custom libc to add support for + // targets that are not yet available upstream. Adding a patch to replace libc with a + // custom one would cause compilation errors though, because Cargo would interpret the + // custom libc as part of the workspace, and apply the check-cfg lints on it. + // + // The libc build script emits check-cfg flags only when this environment variable is set, + // so this line allows the use of custom libcs. + cargo.env("LIBC_CHECK_CFG", "1"); + if source_type == SourceType::InTree { let mut lint_flags = Vec::new(); // When extending this list, add the new lints to the RUSTFLAGS of the @@ -1920,6 +1991,12 @@ impl<'a> Builder<'a> { rustflags.arg("-Zvalidate-mir"); rustflags.arg(&format!("-Zmir-opt-level={}", mir_opt_level)); } + // Always enable inlining MIR when building the standard library. + // Without this flag, MIR inlining is disabled when incremental compilation is enabled. + // That causes some mir-opt tests which inline functions from the standard library to + // break when incremental compilation is enabled. So this overrides the "no inlining + // during incremental builds" heuristic for the standard library. + rustflags.arg("-Zinline-mir"); } Cargo { command: cargo, rustflags, rustdocflags, allow_features } diff --git a/src/bootstrap/cache.rs b/src/bootstrap/cache.rs index 05f25af68..5376c4ec9 100644 --- a/src/bootstrap/cache.rs +++ b/src/bootstrap/cache.rs @@ -1,9 +1,8 @@ use std::any::{Any, TypeId}; use std::borrow::Borrow; use std::cell::RefCell; -use std::cmp::{Ord, Ordering, PartialOrd}; +use std::cmp::Ordering; use std::collections::HashMap; -use std::convert::AsRef; use std::fmt; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs index eae81b9fc..c3e3fa009 100644 --- a/src/bootstrap/channel.rs +++ b/src/bootstrap/channel.rs @@ -19,7 +19,7 @@ pub enum GitInfo { #[default] Absent, /// This is a git repository. - /// If the info should be used (`ignore_git` is false), this will be + /// If the info should be used (`omit_git_hash` is false), this will be /// `Some`, otherwise it will be `None`. Present(Option<Info>), /// This is not a git repostory, but the info can be fetched from the @@ -35,7 +35,7 @@ pub struct Info { } impl GitInfo { - pub fn new(ignore_git: bool, dir: &Path) -> GitInfo { + pub fn new(omit_git_hash: bool, dir: &Path) -> GitInfo { // See if this even begins to look like a git dir if !dir.join(".git").exists() { match read_commit_info_file(dir) { @@ -52,7 +52,7 @@ impl GitInfo { // If we're ignoring the git info, we don't actually need to collect it, just make sure this // was a git repo in the first place. - if ignore_git { + if omit_git_hash { return GitInfo::Present(None); } @@ -139,7 +139,7 @@ pub fn read_commit_info_file(root: &Path) -> Option<Info> { sha: sha.to_owned(), short_sha: short_sha.to_owned(), }, - _ => panic!("the `git-comit-info` file is malformed"), + _ => panic!("the `git-commit-info` file is malformed"), }; Some(info) } else { diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index cd1966713..fcaa69831 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -237,7 +237,7 @@ impl Step for Rustc { target, cargo_subcommand(builder.kind), ); - rustc_cargo(builder, &mut cargo, target); + rustc_cargo(builder, &mut cargo, target, compiler.stage); // For ./x.py clippy, don't run with --all-targets because // linting tests and benchmarks can produce very noisy results @@ -271,9 +271,17 @@ impl Step for Rustc { false, ); - let libdir = builder.sysroot_libdir(compiler, target); - let hostdir = builder.sysroot_libdir(compiler, compiler.host); - add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target)); + // HACK: This avoids putting the newly built artifacts in the sysroot if we're using + // `download-rustc`, to avoid "multiple candidates for `rmeta`" errors. Technically, that's + // not quite right: people can set `download-rustc = true` to download even if there are + // changes to the compiler, and in that case ideally we would put the *new* artifacts in the + // sysroot, in case there are API changes that should be used by tools. In practice, + // though, that should be very uncommon, and people can still disable download-rustc. + if !builder.download_rustc() { + let libdir = builder.sysroot_libdir(compiler, target); + let hostdir = builder.sysroot_libdir(compiler, compiler.host); + add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target)); + } } } @@ -315,7 +323,7 @@ impl Step for CodegenBackend { cargo .arg("--manifest-path") .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend))); - rustc_cargo_env(builder, &mut cargo, target); + rustc_cargo_env(builder, &mut cargo, target, compiler.stage); let msg = if compiler.host == target { format!("Checking stage{} {} artifacts ({target})", builder.top_stage, backend) diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 8b80dfc0f..4a4e7adcb 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -20,11 +20,11 @@ use serde_derive::Deserialize; use crate::builder::crate_description; use crate::builder::Cargo; -use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; use crate::cache::{Interned, INTERNER}; use crate::config::{LlvmLibunwind, RustcLto, TargetSelection}; use crate::dist; -use crate::native; +use crate::llvm; use crate::tool::SourceType; use crate::util::get_clang_cl_resource_dir; use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date}; @@ -83,11 +83,11 @@ impl Step for Std { let target = self.target; let compiler = self.compiler; - // These artifacts were already copied (in `impl Step for Sysroot`). - // Don't recompile them. + // When using `download-rustc`, we already have artifacts for the host available + // (they were copied in `impl Step for Sysroot`). Don't recompile them. // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, // so its artifacts can't be reused. - if builder.download_rustc() && compiler.stage != 0 { + if builder.download_rustc() && compiler.stage != 0 && target == builder.build.build { return; } @@ -191,7 +191,7 @@ fn copy_and_stamp( } fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf { - let libunwind_path = builder.ensure(native::Libunwind { target }); + let libunwind_path = builder.ensure(llvm::Libunwind { target }); let libunwind_source = libunwind_path.join("libunwind.a"); let libunwind_target = libdir.join("libunwind.a"); builder.copy(&libunwind_source, &libunwind_target); @@ -266,7 +266,7 @@ fn copy_self_contained_objects( DependencyType::TargetSelfContained, ); } - let crt_path = builder.ensure(native::CrtBeginEnd { target }); + let crt_path = builder.ensure(llvm::CrtBeginEnd { target }); for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { let src = crt_path.join(obj); let target = libdir_self_contained.join(obj); @@ -339,6 +339,12 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car "" }; + // `libtest` uses this to know whether or not to support + // `-Zunstable-options`. + if !builder.unstable_features() { + cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); + } + let mut features = String::new(); // Cranelift doesn't support `asm`. @@ -468,7 +474,7 @@ fn copy_sanitizers( compiler: &Compiler, target: TargetSelection, ) -> Vec<PathBuf> { - let runtimes: Vec<native::SanitizerRuntime> = builder.ensure(native::Sanitizers { target }); + let runtimes: Vec<llvm::SanitizerRuntime> = builder.ensure(llvm::Sanitizers { target }); if builder.config.dry_run() { return Vec::new(); @@ -690,7 +696,7 @@ impl Step for Rustc { )); let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "build"); - rustc_cargo(builder, &mut cargo, target); + rustc_cargo(builder, &mut cargo, target, compiler.stage); if builder.config.rust_profile_use.is_some() && builder.config.rust_profile_generate.is_some() @@ -807,16 +813,21 @@ impl Step for Rustc { } } -pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { +pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection, stage: u32) { cargo .arg("--features") .arg(builder.rustc_features(builder.kind)) .arg("--manifest-path") .arg(builder.src.join("compiler/rustc/Cargo.toml")); - rustc_cargo_env(builder, cargo, target); + rustc_cargo_env(builder, cargo, target, stage); } -pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { +pub fn rustc_cargo_env( + builder: &Builder<'_>, + cargo: &mut Cargo, + target: TargetSelection, + stage: u32, +) { // Set some configuration variables picked up by build scripts and // the compiler alike cargo @@ -861,83 +872,86 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); } - // Pass down configuration from the LLVM build into the build of - // rustc_llvm and rustc_codegen_llvm. - // // Note that this is disabled if LLVM itself is disabled or we're in a check // build. If we are in a check build we still go ahead here presuming we've // detected that LLVM is already built and good to go which helps prevent // busting caches (e.g. like #71152). - if builder.config.llvm_enabled() - && (builder.kind != Kind::Check - || crate::native::prebuilt_llvm_config(builder, target).is_ok()) - { - if builder.is_rust_llvm(target) { - cargo.env("LLVM_RUSTLLVM", "1"); - } - let native::LlvmResult { llvm_config, .. } = builder.ensure(native::Llvm { target }); - cargo.env("LLVM_CONFIG", &llvm_config); - if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { - cargo.env("CFG_LLVM_ROOT", s); + if builder.config.llvm_enabled() { + let building_is_expensive = crate::llvm::prebuilt_llvm_config(builder, target).is_err(); + // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler + let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; + let should_skip_build = building_is_expensive && can_skip_build; + if !should_skip_build { + rustc_llvm_env(builder, cargo, target) } + } +} - // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script - // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by - // whitespace. - // - // For example: - // - on windows, when `clang-cl` is used with instrumentation, we need to manually add - // clang's runtime library resource directory so that the profiler runtime library can be - // found. This is to avoid the linker errors about undefined references to - // `__llvm_profile_instrument_memop` when linking `rustc_driver`. - let mut llvm_linker_flags = String::new(); - if builder.config.llvm_profile_generate && target.contains("msvc") { - if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { - // Add clang's runtime library directory to the search path - let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); - llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); - } - } +/// Pass down configuration from the LLVM build into the build of +/// rustc_llvm and rustc_codegen_llvm. +fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { + let target_config = builder.config.target_config.get(&target); - // The config can also specify its own llvm linker flags. - if let Some(ref s) = builder.config.llvm_ldflags { - if !llvm_linker_flags.is_empty() { - llvm_linker_flags.push_str(" "); - } - llvm_linker_flags.push_str(s); + if builder.is_rust_llvm(target) { + cargo.env("LLVM_RUSTLLVM", "1"); + } + let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target }); + cargo.env("LLVM_CONFIG", &llvm_config); + if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + cargo.env("CFG_LLVM_ROOT", s); + } + + // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script + // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by + // whitespace. + // + // For example: + // - on windows, when `clang-cl` is used with instrumentation, we need to manually add + // clang's runtime library resource directory so that the profiler runtime library can be + // found. This is to avoid the linker errors about undefined references to + // `__llvm_profile_instrument_memop` when linking `rustc_driver`. + let mut llvm_linker_flags = String::new(); + if builder.config.llvm_profile_generate && target.contains("msvc") { + if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { + // Add clang's runtime library directory to the search path + let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); + llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); } + } - // Set the linker flags via the env var that `rustc_llvm`'s build script will read. + // The config can also specify its own llvm linker flags. + if let Some(ref s) = builder.config.llvm_ldflags { if !llvm_linker_flags.is_empty() { - cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); + llvm_linker_flags.push_str(" "); } + llvm_linker_flags.push_str(s); + } - // Building with a static libstdc++ is only supported on linux right now, - // not for MSVC or macOS - if builder.config.llvm_static_stdcpp - && !target.contains("freebsd") - && !target.contains("msvc") - && !target.contains("apple") - && !target.contains("solaris") - { - let file = compiler_file( - builder, - builder.cxx(target).unwrap(), - target, - CLang::Cxx, - "libstdc++.a", - ); - cargo.env("LLVM_STATIC_STDCPP", file); - } - if builder.llvm_link_shared() { - cargo.env("LLVM_LINK_SHARED", "1"); - } - if builder.config.llvm_use_libcxx { - cargo.env("LLVM_USE_LIBCXX", "1"); - } - if builder.config.llvm_optimize && !builder.config.llvm_release_debuginfo { - cargo.env("LLVM_NDEBUG", "1"); - } + // Set the linker flags via the env var that `rustc_llvm`'s build script will read. + if !llvm_linker_flags.is_empty() { + cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); + } + + // Building with a static libstdc++ is only supported on linux right now, + // not for MSVC or macOS + if builder.config.llvm_static_stdcpp + && !target.contains("freebsd") + && !target.contains("msvc") + && !target.contains("apple") + && !target.contains("solaris") + { + let file = + compiler_file(builder, builder.cxx(target).unwrap(), target, CLang::Cxx, "libstdc++.a"); + cargo.env("LLVM_STATIC_STDCPP", file); + } + if builder.llvm_link_shared() { + cargo.env("LLVM_LINK_SHARED", "1"); + } + if builder.config.llvm_use_libcxx { + cargo.env("LLVM_USE_LIBCXX", "1"); + } + if builder.config.llvm_optimize && !builder.config.llvm_release_debuginfo { + cargo.env("LLVM_NDEBUG", "1"); } } @@ -989,6 +1003,44 @@ pub struct CodegenBackend { pub backend: Interned<String>, } +fn needs_codegen_config(run: &RunConfig<'_>) -> bool { + let mut needs_codegen_cfg = false; + for path_set in &run.paths { + needs_codegen_cfg = match path_set { + PathSet::Set(set) => set.iter().any(|p| is_codegen_cfg_needed(p, run)), + PathSet::Suite(suite) => is_codegen_cfg_needed(&suite, run), + } + } + needs_codegen_cfg +} + +const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; + +fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { + if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) { + let mut needs_codegen_backend_config = true; + for &backend in &run.builder.config.rust_codegen_backends { + if path + .path + .to_str() + .unwrap() + .ends_with(&(CODEGEN_BACKEND_PREFIX.to_owned() + &backend)) + { + needs_codegen_backend_config = false; + } + } + if needs_codegen_backend_config { + run.builder.info( + "Warning: no codegen-backends config matched the requested path to build a codegen backend. \ + Help: add backend to codegen-backends in config.toml.", + ); + return true; + } + } + + return false; +} + impl Step for CodegenBackend { type Output = (); const ONLY_HOSTS: bool = true; @@ -1000,6 +1052,10 @@ impl Step for CodegenBackend { } fn make_run(run: RunConfig<'_>) { + if needs_codegen_config(&run) { + return; + } + for &backend in &run.builder.config.rust_codegen_backends { if backend == "llvm" { continue; // Already built as part of rustc @@ -1042,7 +1098,7 @@ impl Step for CodegenBackend { cargo .arg("--manifest-path") .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend))); - rustc_cargo_env(builder, &mut cargo, target); + rustc_cargo_env(builder, &mut cargo, target, compiler.stage); let tmp_stamp = out_dir.join(".tmp.stamp"); @@ -1225,9 +1281,7 @@ impl Step for Sysroot { } // Copy the compiler into the correct sysroot. - let ci_rustc_dir = - builder.config.out.join(&*builder.config.build.triple).join("ci-rustc"); - builder.cp_r(&ci_rustc_dir, &sysroot); + builder.cp_r(&builder.ci_rustc_dir(builder.build.build), &sysroot); return INTERNER.intern_path(sysroot); } @@ -1329,7 +1383,10 @@ impl Step for Assemble { // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. if builder.download_rustc() { - builder.ensure(Sysroot { compiler: target_compiler }); + let sysroot = builder.ensure(Sysroot { compiler: target_compiler }); + // Ensure that `libLLVM.so` ends up in the newly created target directory, + // so that tools using `rustc_private` can use it. + dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); return target_compiler; } @@ -1340,6 +1397,13 @@ impl Step for Assemble { // when not performing a full bootstrap). builder.ensure(Rustc::new(build_compiler, target_compiler.host)); + // FIXME: For now patch over problems noted in #90244 by early returning here, even though + // we've not properly assembled the target sysroot. A full fix is pending further investigation, + // for now full bootstrap usage is rare enough that this is OK. + if target_compiler.stage >= 3 && !builder.config.full_bootstrap { + return target_compiler; + } + for &backend in builder.config.rust_codegen_backends.iter() { if backend == "llvm" { continue; // Already built as part of rustc @@ -1353,7 +1417,7 @@ impl Step for Assemble { } let lld_install = if builder.config.lld_enabled { - Some(builder.ensure(native::Lld { target: target_compiler.host })) + Some(builder.ensure(llvm::Lld { target: target_compiler.host })) } else { None }; @@ -1417,8 +1481,8 @@ impl Step for Assemble { } if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { - let native::LlvmResult { llvm_config, .. } = - builder.ensure(native::Llvm { target: target_compiler.host }); + let llvm::LlvmResult { llvm_config, .. } = + builder.ensure(llvm::Llvm { target: target_compiler.host }); if !builder.config.dry_run() { let llvm_bin_dir = output(Command::new(llvm_config).arg("--bindir")); let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 05e742549..cc3b3bc25 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -55,9 +55,8 @@ pub enum DryRun { /// Note that this structure is not decoded directly into, but rather it is /// filled out from the decoded forms of the structs below. For documentation /// each field, see the corresponding fields in -/// `config.toml.example`. -#[derive(Default)] -#[cfg_attr(test, derive(Clone))] +/// `config.example.toml`. +#[derive(Default, Clone)] pub struct Config { pub changelog_seen: Option<usize>, pub ccache: Option<String>, @@ -77,7 +76,7 @@ pub struct Config { pub tools: Option<HashSet<String>>, pub sanitizers: bool, pub profiler: bool, - pub ignore_git: bool, + pub omit_git_hash: bool, pub exclude: Vec<TaskPath>, pub include_default_paths: bool, pub rustc_error_format: Option<String>, @@ -87,6 +86,9 @@ pub struct Config { pub patch_binaries_for_nix: bool, pub stage0_metadata: Stage0Metadata, + pub stdout_is_tty: bool, + pub stderr_is_tty: bool, + pub on_fail: Option<String>, pub stage: u32, pub keep_stage: Vec<u32>, @@ -112,14 +114,12 @@ pub struct Config { pub backtrace_on_ice: bool, // llvm codegen options - pub llvm_skip_rebuild: bool, pub llvm_assertions: bool, pub llvm_tests: bool, pub llvm_plugins: bool, pub llvm_optimize: bool, pub llvm_thin_lto: bool, pub llvm_release_debuginfo: bool, - pub llvm_version_check: bool, pub llvm_static_stdcpp: bool, /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. #[cfg(not(test))] @@ -135,6 +135,7 @@ pub struct Config { pub llvm_allow_old_toolchain: bool, pub llvm_polly: bool, pub llvm_clang: bool, + pub llvm_enable_warnings: bool, pub llvm_from_ci: bool, pub llvm_build_config: HashMap<String, String>, @@ -192,6 +193,8 @@ pub struct Config { pub dist_sign_folder: Option<PathBuf>, pub dist_upload_addr: Option<String>, pub dist_compression_formats: Option<Vec<String>>, + pub dist_compression_profile: String, + pub dist_include_mingw_linker: bool, // libstd features pub backtrace: bool, // support for RUST_BACKTRACE @@ -223,27 +226,33 @@ pub struct Config { pub reuse: Option<PathBuf>, pub cargo_native_static: bool, pub configure_args: Vec<String>, + pub out: PathBuf, + pub rust_info: channel::GitInfo, // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, + #[cfg(not(test))] initial_rustfmt: RefCell<RustfmtState>, #[cfg(test)] pub initial_rustfmt: RefCell<RustfmtState>, - pub out: PathBuf, - pub rust_info: channel::GitInfo, } -#[derive(Default, Deserialize)] -#[cfg_attr(test, derive(Clone))] +#[derive(Default, Deserialize, Clone)] pub struct Stage0Metadata { + pub compiler: CompilerMetadata, pub config: Stage0Config, pub checksums_sha256: HashMap<String, String>, pub rustfmt: Option<RustfmtMetadata>, } -#[derive(Default, Deserialize)] -#[cfg_attr(test, derive(Clone))] +#[derive(Default, Deserialize, Clone)] +pub struct CompilerMetadata { + pub date: String, + pub version: String, +} + +#[derive(Default, Deserialize, Clone)] pub struct Stage0Config { pub dist_server: String, pub artifacts_server: String, @@ -251,8 +260,7 @@ pub struct Stage0Config { pub git_merge_commit_email: String, pub nightly_branch: String, } -#[derive(Default, Deserialize)] -#[cfg_attr(test, derive(Clone))] +#[derive(Default, Deserialize, Clone)] pub struct RustfmtMetadata { pub date: String, pub version: String, @@ -326,7 +334,7 @@ impl std::str::FromStr for SplitDebuginfo { impl SplitDebuginfo { /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for - /// `rust.split-debuginfo` in `config.toml.example`. + /// `rust.split-debuginfo` in `config.example.toml`. fn default_for_platform(target: &str) -> Self { if target.contains("apple") { SplitDebuginfo::Unpacked @@ -430,8 +438,7 @@ impl PartialEq<&str> for TargetSelection { } /// Per-target configuration stored in the global configuration structure. -#[derive(Default)] -#[cfg_attr(test, derive(Clone))] +#[derive(Default, Clone)] pub struct Target { /// Some(path to llvm-config) if using an external LLVM. pub llvm_config: Option<PathBuf>, @@ -666,7 +673,6 @@ define_config! { define_config! { /// TOML representation of how the LLVM build is configured. struct Llvm { - skip_rebuild: Option<bool> = "skip-rebuild", optimize: Option<bool> = "optimize", thin_lto: Option<bool> = "thin-lto", release_debuginfo: Option<bool> = "release-debuginfo", @@ -674,7 +680,6 @@ define_config! { tests: Option<bool> = "tests", plugins: Option<bool> = "plugins", ccache: Option<StringOrBool> = "ccache", - version_check: Option<bool> = "version-check", static_libstdcpp: Option<bool> = "static-libstdcpp", ninja: Option<bool> = "ninja", targets: Option<String> = "targets", @@ -691,6 +696,7 @@ define_config! { allow_old_toolchain: Option<bool> = "allow-old-toolchain", polly: Option<bool> = "polly", clang: Option<bool> = "clang", + enable_warnings: Option<bool> = "enable-warnings", download_ci_llvm: Option<StringOrBool> = "download-ci-llvm", build_config: Option<HashMap<String, String>> = "build-config", } @@ -704,6 +710,8 @@ define_config! { src_tarball: Option<bool> = "src-tarball", missing_tools: Option<bool> = "missing-tools", compression_formats: Option<Vec<String>> = "compression-formats", + compression_profile: Option<String> = "compression-profile", + include_mingw_linker: Option<bool> = "include-mingw-linker", } } @@ -750,7 +758,7 @@ define_config! { verbose_tests: Option<bool> = "verbose-tests", optimize_tests: Option<bool> = "optimize-tests", codegen_tests: Option<bool> = "codegen-tests", - ignore_git: Option<bool> = "ignore-git", + omit_git_hash: Option<bool> = "omit-git-hash", dist_src: Option<bool> = "dist-src", save_toolstates: Option<String> = "save-toolstates", codegen_backends: Option<Vec<String>> = "codegen-backends", @@ -803,10 +811,11 @@ define_config! { impl Config { pub fn default_opts() -> Config { + use is_terminal::IsTerminal; + let mut config = Config::default(); config.llvm_optimize = true; config.ninja_in_file = true; - config.llvm_version_check = true; config.llvm_static_stdcpp = false; config.backtrace = true; config.rust_optimize = true; @@ -821,6 +830,11 @@ impl Config { config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")]; config.deny_warnings = true; config.bindir = "bin".into(); + config.dist_include_mingw_linker = true; + config.dist_compression_profile = "fast".into(); + + config.stdout_is_tty = std::io::stdout().is_terminal(); + config.stderr_is_tty = std::io::stderr().is_terminal(); // set by build.rs config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); @@ -989,10 +1003,10 @@ impl Config { config.out = crate::util::absolute(&config.out); } - config.initial_rustc = build - .rustc - .map(PathBuf::from) - .unwrap_or_else(|| config.out.join(config.build.triple).join("stage0/bin/rustc")); + config.initial_rustc = build.rustc.map(PathBuf::from).unwrap_or_else(|| { + config.download_beta_toolchain(); + config.out.join(config.build.triple).join("stage0/bin/rustc") + }); config.initial_cargo = build .cargo .map(PathBuf::from) @@ -1060,11 +1074,6 @@ impl Config { config.mandir = install.mandir.map(PathBuf::from); } - // We want the llvm-skip-rebuild flag to take precedence over the - // skip-rebuild config.toml option so we store it separately - // so that we can infer the right value - let mut llvm_skip_rebuild = flags.llvm_skip_rebuild; - // Store off these values as options because if they're not provided // we'll infer default values for them later let mut llvm_assertions = None; @@ -1082,7 +1091,7 @@ impl Config { let mut debuginfo_level_tools = None; let mut debuginfo_level_tests = None; let mut optimize = None; - let mut ignore_git = None; + let mut omit_git_hash = None; if let Some(rust) = toml.rust { debug = rust.debug; @@ -1103,7 +1112,7 @@ impl Config { .map(|v| v.expect("invalid value for rust.split_debuginfo")) .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple)); optimize = rust.optimize; - ignore_git = rust.ignore_git; + omit_git_hash = rust.omit_git_hash; config.rust_new_symbol_mangling = rust.new_symbol_mangling; set(&mut config.rust_optimize_tests, rust.optimize_tests); set(&mut config.codegen_tests, rust.codegen_tests); @@ -1158,6 +1167,11 @@ impl Config { config.rust_profile_generate = flags.rust_profile_generate; } + // rust_info must be set before is_ci_llvm_available() is called. + let default = config.channel == "dev"; + config.omit_git_hash = omit_git_hash.unwrap_or(default); + config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); + if let Some(llvm) = toml.llvm { match llvm.ccache { Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), @@ -1170,11 +1184,9 @@ impl Config { llvm_assertions = llvm.assertions; llvm_tests = llvm.tests; llvm_plugins = llvm.plugins; - llvm_skip_rebuild = llvm_skip_rebuild.or(llvm.skip_rebuild); set(&mut config.llvm_optimize, llvm.optimize); set(&mut config.llvm_thin_lto, llvm.thin_lto); set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo); - set(&mut config.llvm_version_check, llvm.version_check); set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp); if let Some(v) = llvm.link_shared { config.llvm_link_shared.set(Some(v)); @@ -1193,17 +1205,18 @@ impl Config { config.llvm_allow_old_toolchain = llvm.allow_old_toolchain.unwrap_or(false); config.llvm_polly = llvm.polly.unwrap_or(false); config.llvm_clang = llvm.clang.unwrap_or(false); + config.llvm_enable_warnings = llvm.enable_warnings.unwrap_or(false); config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default()); let asserts = llvm_assertions.unwrap_or(false); config.llvm_from_ci = match llvm.download_ci_llvm { Some(StringOrBool::String(s)) => { assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s); - crate::native::is_ci_llvm_available(&config, asserts) + crate::llvm::is_ci_llvm_available(&config, asserts) } Some(StringOrBool::Bool(b)) => b, None => { - config.channel == "dev" && crate::native::is_ci_llvm_available(&config, asserts) + config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, asserts) } }; @@ -1246,7 +1259,7 @@ impl Config { } } else { config.llvm_from_ci = - config.channel == "dev" && crate::native::is_ci_llvm_available(&config, false); + config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, false); } if let Some(t) = toml.target { @@ -1309,8 +1322,10 @@ impl Config { config.dist_sign_folder = t.sign_folder.map(PathBuf::from); config.dist_upload_addr = t.upload_addr; config.dist_compression_formats = t.compression_formats; + set(&mut config.dist_compression_profile, t.compression_profile); set(&mut config.rust_dist_src, t.src_tarball); set(&mut config.missing_tools, t.missing_tools); + set(&mut config.dist_include_mingw_linker, t.include_mingw_linker) } if let Some(r) = build.rustfmt { @@ -1324,7 +1339,6 @@ impl Config { // Now that we've reached the end of our configuration, infer the // default values for all options that we haven't otherwise stored yet. - config.llvm_skip_rebuild = llvm_skip_rebuild.unwrap_or(false); config.llvm_assertions = llvm_assertions.unwrap_or(false); config.llvm_tests = llvm_tests.unwrap_or(false); config.llvm_plugins = llvm_plugins.unwrap_or(false); @@ -1352,10 +1366,6 @@ impl Config { config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(0); - let default = config.channel == "dev"; - config.ignore_git = ignore_git.unwrap_or(default); - config.rust_info = GitInfo::new(config.ignore_git, &config.src); - let download_rustc = config.download_rustc_commit.is_some(); // See https://github.com/rust-lang/compiler-team/issues/326 config.stage = match config.cmd { @@ -1380,7 +1390,8 @@ impl Config { | Subcommand::Fix { .. } | Subcommand::Run { .. } | Subcommand::Setup { .. } - | Subcommand::Format { .. } => flags.stage.unwrap_or(0), + | Subcommand::Format { .. } + | Subcommand::Suggest { .. } => flags.stage.unwrap_or(0), }; // CI should always run stage 2 builds, unless it specifically states otherwise @@ -1405,7 +1416,8 @@ impl Config { | Subcommand::Fix { .. } | Subcommand::Run { .. } | Subcommand::Setup { .. } - | Subcommand::Format { .. } => {} + | Subcommand::Format { .. } + | Subcommand::Suggest { .. } => {} } } diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/config/tests.rs index 5a105007f..50569eb4f 100644 --- a/src/bootstrap/config/tests.rs +++ b/src/bootstrap/config/tests.rs @@ -1,5 +1,5 @@ use super::{Config, TomlConfig}; -use std::path::Path; +use std::{env, path::Path}; fn toml(config: &str) -> impl '_ + Fn(&Path) -> TomlConfig { |&_| toml::from_str(config).unwrap() @@ -11,7 +11,7 @@ fn parse(config: &str) -> Config { #[test] fn download_ci_llvm() { - if crate::native::is_ci_llvm_modified(&parse("")) { + if crate::llvm::is_ci_llvm_modified(&parse("")) { eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); return; } @@ -33,4 +33,58 @@ fn download_ci_llvm() { )); } -// FIXME: add test for detecting `src` and `out` +// FIXME(ozkanonur): extend scope of the test +// refs: +// - https://github.com/rust-lang/rust/issues/109120 +// - https://github.com/rust-lang/rust/pull/109162#issuecomment-1496782487 +#[test] +fn detect_src_and_out() { + fn test(cfg: Config, build_dir: Option<&str>) { + // This will bring absolute form of `src/bootstrap` path + let current_dir = std::env::current_dir().unwrap(); + + // get `src` by moving into project root path + let expected_src = current_dir.ancestors().nth(2).unwrap(); + assert_eq!(&cfg.src, expected_src); + + // Sanity check for `src` + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let expected_src = manifest_dir.ancestors().nth(2).unwrap(); + assert_eq!(&cfg.src, expected_src); + + // test if build-dir was manually given in config.toml + if let Some(custom_build_dir) = build_dir { + assert_eq!(&cfg.out, Path::new(custom_build_dir)); + } + // test the native bootstrap way + else { + // This should bring output path of bootstrap in absolute form + let cargo_target_dir = env::var_os("CARGO_TARGET_DIR").expect( + "CARGO_TARGET_DIR must been provided for the test environment from bootstrap", + ); + + // Move to `build` from `build/bootstrap` + let expected_out = Path::new(&cargo_target_dir).parent().unwrap(); + assert_eq!(&cfg.out, expected_out); + + let args: Vec<String> = env::args().collect(); + + // Another test for `out` as a sanity check + // + // This will bring something similar to: + // `{build-dir}/bootstrap/debug/deps/bootstrap-c7ee91d5661e2804` + // `{build-dir}` can be anywhere, not just in the rust project directory. + let dep = Path::new(args.first().unwrap()); + let expected_out = dep.ancestors().nth(4).unwrap(); + + assert_eq!(&cfg.out, expected_out); + } + } + + test(parse(""), None); + + { + let build_dir = if cfg!(windows) { Some("C:\\tmp") } else { Some("/tmp") }; + test(parse("build.build-dir = \"/tmp\""), build_dir); + } +} diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index ab3d08292..abd28b400 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -44,7 +44,6 @@ o("local-rebuild", "build.local-rebuild", "assume local-rust matches the current o("llvm-static-stdcpp", "llvm.static-libstdcpp", "statically link to libstdc++ for LLVM") o("llvm-link-shared", "llvm.link-shared", "prefer shared linking to LLVM (llvm-config --link-shared)") o("rpath", "rust.rpath", "build rpaths into rustc itself") -o("llvm-version-check", "llvm.version-check", "check if the LLVM version is supported, build anyway") o("codegen-tests", "rust.codegen-tests", "run the tests/codegen tests") o("option-checking", None, "complain about unrecognized options in this configure script") o("ninja", "llvm.ninja", "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)") @@ -195,7 +194,7 @@ if '--help' in sys.argv or '-h' in sys.argv: print('') print('This configure script is a thin configuration shim over the true') print('configuration system, `config.toml`. You can explore the comments') - print('in `config.toml.example` next to this configure script to see') + print('in `config.example.toml` next to this configure script to see') print('more information about what each option is. Additionally you can') print('pass `--set` as an argument to set arbitrary key/value pairs') print('in the TOML configuration if desired') @@ -206,77 +205,78 @@ if '--help' in sys.argv or '-h' in sys.argv: # Parse all command line arguments into one of these three lists, handling # boolean and value-based options separately -unknown_args = [] -need_value_args = [] -known_args = {} - -p("processing command line") -i = 1 -while i < len(sys.argv): - arg = sys.argv[i] - i += 1 - if not arg.startswith('--'): - unknown_args.append(arg) - continue - - found = False - for option in options: - value = None - if option.value: - keyval = arg[2:].split('=', 1) - key = keyval[0] - if option.name != key: - continue +def parse_args(args): + unknown_args = [] + need_value_args = [] + known_args = {} + + i = 0 + while i < len(args): + arg = args[i] + i += 1 + if not arg.startswith('--'): + unknown_args.append(arg) + continue - if len(keyval) > 1: - value = keyval[1] - elif i < len(sys.argv): - value = sys.argv[i] - i += 1 - else: - need_value_args.append(arg) - continue - else: - if arg[2:] == 'enable-' + option.name: - value = True - elif arg[2:] == 'disable-' + option.name: - value = False + found = False + for option in options: + value = None + if option.value: + keyval = arg[2:].split('=', 1) + key = keyval[0] + if option.name != key: + continue + + if len(keyval) > 1: + value = keyval[1] + elif i < len(args): + value = args[i] + i += 1 + else: + need_value_args.append(arg) + continue else: - continue + if arg[2:] == 'enable-' + option.name: + value = True + elif arg[2:] == 'disable-' + option.name: + value = False + else: + continue + + found = True + if option.name not in known_args: + known_args[option.name] = [] + known_args[option.name].append((option, value)) + break + + if not found: + unknown_args.append(arg) + + # Note: here and a few other places, we use [-1] to apply the *last* value + # passed. But if option-checking is enabled, then the known_args loop will + # also assert that options are only passed once. + option_checking = ('option-checking' not in known_args + or known_args['option-checking'][-1][1]) + if option_checking: + if len(unknown_args) > 0: + err("Option '" + unknown_args[0] + "' is not recognized") + if len(need_value_args) > 0: + err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0])) + + config = {} + + set('build.configure-args', sys.argv[1:], config) + apply_args(known_args, option_checking, config) + return parse_example_config(known_args, config) - found = True - if option.name not in known_args: - known_args[option.name] = [] - known_args[option.name].append((option, value)) - break - - if not found: - unknown_args.append(arg) -p("") - -# Note: here and a few other places, we use [-1] to apply the *last* value -# passed. But if option-checking is enabled, then the known_args loop will -# also assert that options are only passed once. -option_checking = ('option-checking' not in known_args - or known_args['option-checking'][-1][1]) -if option_checking: - if len(unknown_args) > 0: - err("Option '" + unknown_args[0] + "' is not recognized") - if len(need_value_args) > 0: - err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0])) - -# Parse all known arguments into a configuration structure that reflects the -# TOML we're going to write out -config = {} - - -def build(): + +def build(known_args): if 'build' in known_args: return known_args['build'][-1][1] return bootstrap.default_build_triple(verbose=False) -def set(key, value): +def set(key, value, config): if isinstance(value, list): # Remove empty values, which value.split(',') tends to generate. value = [v for v in value if v] @@ -298,122 +298,127 @@ def set(key, value): arr = arr[part] -for key in known_args: - # The `set` option is special and can be passed a bunch of times - if key == 'set': - for option, value in known_args[key]: - keyval = value.split('=', 1) - if len(keyval) == 1 or keyval[1] == "true": - value = True - elif keyval[1] == "false": - value = False - else: - value = keyval[1] - set(keyval[0], value) - continue - - # Ensure each option is only passed once - arr = known_args[key] - if option_checking and len(arr) > 1: - err("Option '{}' provided more than once".format(key)) - option, value = arr[-1] - - # If we have a clear avenue to set our value in rustbuild, do so - if option.rustbuild is not None: - set(option.rustbuild, value) - continue - - # Otherwise we're a "special" option and need some extra handling, so do - # that here. - if option.name == 'sccache': - set('llvm.ccache', 'sccache') - elif option.name == 'local-rust': - for path in os.environ['PATH'].split(os.pathsep): - if os.path.exists(path + '/rustc'): - set('build.rustc', path + '/rustc') - break - for path in os.environ['PATH'].split(os.pathsep): - if os.path.exists(path + '/cargo'): - set('build.cargo', path + '/cargo') - break - elif option.name == 'local-rust-root': - set('build.rustc', value + '/bin/rustc') - set('build.cargo', value + '/bin/cargo') - elif option.name == 'llvm-root': - set('target.{}.llvm-config'.format(build()), value + '/bin/llvm-config') - elif option.name == 'llvm-config': - set('target.{}.llvm-config'.format(build()), value) - elif option.name == 'llvm-filecheck': - set('target.{}.llvm-filecheck'.format(build()), value) - elif option.name == 'tools': - set('build.tools', value.split(',')) - elif option.name == 'codegen-backends': - set('rust.codegen-backends', value.split(',')) - elif option.name == 'host': - set('build.host', value.split(',')) - elif option.name == 'target': - set('build.target', value.split(',')) - elif option.name == 'full-tools': - set('rust.codegen-backends', ['llvm']) - set('rust.lld', True) - set('rust.llvm-tools', True) - set('build.extended', True) - elif option.name == 'option-checking': - # this was handled above - pass - elif option.name == 'dist-compression-formats': - set('dist.compression-formats', value.split(',')) - else: - raise RuntimeError("unhandled option {}".format(option.name)) +def apply_args(known_args, option_checking, config): + for key in known_args: + # The `set` option is special and can be passed a bunch of times + if key == 'set': + for option, value in known_args[key]: + keyval = value.split('=', 1) + if len(keyval) == 1 or keyval[1] == "true": + value = True + elif keyval[1] == "false": + value = False + else: + value = keyval[1] + set(keyval[0], value, config) + continue -set('build.configure-args', sys.argv[1:]) + # Ensure each option is only passed once + arr = known_args[key] + if option_checking and len(arr) > 1: + err("Option '{}' provided more than once".format(key)) + option, value = arr[-1] -# "Parse" the `config.toml.example` file into the various sections, and we'll + # If we have a clear avenue to set our value in rustbuild, do so + if option.rustbuild is not None: + set(option.rustbuild, value, config) + continue + + # Otherwise we're a "special" option and need some extra handling, so do + # that here. + build_triple = build(known_args) + + if option.name == 'sccache': + set('llvm.ccache', 'sccache', config) + elif option.name == 'local-rust': + for path in os.environ['PATH'].split(os.pathsep): + if os.path.exists(path + '/rustc'): + set('build.rustc', path + '/rustc', config) + break + for path in os.environ['PATH'].split(os.pathsep): + if os.path.exists(path + '/cargo'): + set('build.cargo', path + '/cargo', config) + break + elif option.name == 'local-rust-root': + set('build.rustc', value + '/bin/rustc', config) + set('build.cargo', value + '/bin/cargo', config) + elif option.name == 'llvm-root': + set('target.{}.llvm-config'.format(build_triple), value + '/bin/llvm-config', config) + elif option.name == 'llvm-config': + set('target.{}.llvm-config'.format(build_triple), value, config) + elif option.name == 'llvm-filecheck': + set('target.{}.llvm-filecheck'.format(build_triple), value, config) + elif option.name == 'tools': + set('build.tools', value.split(','), config) + elif option.name == 'codegen-backends': + set('rust.codegen-backends', value.split(','), config) + elif option.name == 'host': + set('build.host', value.split(','), config) + elif option.name == 'target': + set('build.target', value.split(','), config) + elif option.name == 'full-tools': + set('rust.codegen-backends', ['llvm'], config) + set('rust.lld', True, config) + set('rust.llvm-tools', True, config) + set('build.extended', True, config) + elif option.name == 'option-checking': + # this was handled above + pass + elif option.name == 'dist-compression-formats': + set('dist.compression-formats', value.split(','), config) + else: + raise RuntimeError("unhandled option {}".format(option.name)) + +# "Parse" the `config.example.toml` file into the various sections, and we'll # use this as a template of a `config.toml` to write out which preserves # all the various comments and whatnot. # # Note that the `target` section is handled separately as we'll duplicate it # per configured target, so there's a bit of special handling for that here. -sections = {} -cur_section = None -sections[None] = [] -section_order = [None] -targets = {} -top_level_keys = [] - -for line in open(rust_dir + '/config.toml.example').read().split("\n"): - if cur_section == None: - if line.count('=') == 1: - top_level_key = line.split('=')[0] - top_level_key = top_level_key.strip(' #') - top_level_keys.append(top_level_key) - if line.startswith('['): - cur_section = line[1:-1] - if cur_section.startswith('target'): - cur_section = 'target' - elif '.' in cur_section: - raise RuntimeError("don't know how to deal with section: {}".format(cur_section)) - sections[cur_section] = [line] - section_order.append(cur_section) - else: - sections[cur_section].append(line) - -# Fill out the `targets` array by giving all configured targets a copy of the -# `target` section we just loaded from the example config -configured_targets = [build()] -if 'build' in config: - if 'host' in config['build']: - configured_targets += config['build']['host'] - if 'target' in config['build']: - configured_targets += config['build']['target'] -if 'target' in config: - for target in config['target']: - configured_targets.append(target) -for target in configured_targets: - targets[target] = sections['target'][:] - # For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target. - # Avoid using quotes unless it's necessary. - targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target) if "." in target else target) +def parse_example_config(known_args, config): + sections = {} + cur_section = None + sections[None] = [] + section_order = [None] + targets = {} + top_level_keys = [] + + for line in open(rust_dir + '/config.example.toml').read().split("\n"): + if cur_section == None: + if line.count('=') == 1: + top_level_key = line.split('=')[0] + top_level_key = top_level_key.strip(' #') + top_level_keys.append(top_level_key) + if line.startswith('['): + cur_section = line[1:-1] + if cur_section.startswith('target'): + cur_section = 'target' + elif '.' in cur_section: + raise RuntimeError("don't know how to deal with section: {}".format(cur_section)) + sections[cur_section] = [line] + section_order.append(cur_section) + else: + sections[cur_section].append(line) + + # Fill out the `targets` array by giving all configured targets a copy of the + # `target` section we just loaded from the example config + configured_targets = [build(known_args)] + if 'build' in config: + if 'host' in config['build']: + configured_targets += config['build']['host'] + if 'target' in config['build']: + configured_targets += config['build']['target'] + if 'target' in config: + for target in config['target']: + configured_targets.append(target) + for target in configured_targets: + targets[target] = sections['target'][:] + # For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target. + # Avoid using quotes unless it's necessary. + targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target) if "." in target else target) + + configure_file(sections, top_level_keys, targets, config) + return section_order, sections, targets def is_number(value): @@ -476,38 +481,67 @@ def configure_top_level_key(lines, top_level_key, value): raise RuntimeError("failed to find config line for {}".format(top_level_key)) -for section_key, section_config in config.items(): - if section_key not in sections and section_key not in top_level_keys: - raise RuntimeError("config key {} not in sections or top_level_keys".format(section_key)) - if section_key in top_level_keys: - configure_top_level_key(sections[None], section_key, section_config) +# Modify `sections` to reflect the parsed arguments and example configs. +def configure_file(sections, top_level_keys, targets, config): + for section_key, section_config in config.items(): + if section_key not in sections and section_key not in top_level_keys: + raise RuntimeError("config key {} not in sections or top_level_keys".format(section_key)) + if section_key in top_level_keys: + configure_top_level_key(sections[None], section_key, section_config) + + elif section_key == 'target': + for target in section_config: + configure_section(targets[target], section_config[target]) + else: + configure_section(sections[section_key], section_config) + + +def write_uncommented(target, f): + block = [] + is_comment = True + + for line in target: + block.append(line) + if len(line) == 0: + if not is_comment: + for l in block: + f.write(l + "\n") + block = [] + is_comment = True + continue + is_comment = is_comment and line.startswith('#') + return f - elif section_key == 'target': - for target in section_config: - configure_section(targets[target], section_config[target]) - else: - configure_section(sections[section_key], section_config) -# Now that we've built up our `config.toml`, write it all out in the same -# order that we read it in. -p("") -p("writing `config.toml` in current directory") -with bootstrap.output('config.toml') as f: +def write_config_toml(writer, section_order, targets, sections): for section in section_order: if section == 'target': for target in targets: - for line in targets[target]: - f.write(line + "\n") + writer = write_uncommented(targets[target], writer) else: - for line in sections[section]: - f.write(line + "\n") - -with bootstrap.output('Makefile') as f: - contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in') - contents = open(contents).read() - contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/') - contents = contents.replace("$(CFG_PYTHON)", sys.executable) - f.write(contents) - -p("") -p("run `python {}/x.py --help`".format(rust_dir)) + writer = write_uncommented(sections[section], writer) + + +if __name__ == "__main__": + p("processing command line") + # Parse all known arguments into a configuration structure that reflects the + # TOML we're going to write out + p("") + section_order, sections, targets = parse_args(sys.argv[1:]) + + # Now that we've built up our `config.toml`, write it all out in the same + # order that we read it in. + p("") + p("writing `config.toml` in current directory") + with bootstrap.output('config.toml') as f: + write_config_toml(f, section_order, targets, sections) + + with bootstrap.output('Makefile') as f: + contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in') + contents = open(contents).read() + contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/') + contents = contents.replace("$(CFG_PYTHON)", sys.executable) + f.write(contents) + + p("") + p("run `python {}/x.py --help`".format(rust_dir)) diff --git a/src/bootstrap/defaults/config.codegen.toml b/src/bootstrap/defaults/config.codegen.toml index 088cbd105..113df88d7 100644 --- a/src/bootstrap/defaults/config.codegen.toml +++ b/src/bootstrap/defaults/config.codegen.toml @@ -7,6 +7,10 @@ compiler-docs = true # This enables debug-assertions in LLVM, # catching logic errors in codegen much earlier in the process. assertions = true +# enable warnings during the llvm compilation +enable-warnings = true +# build llvm from source +download-ci-llvm = false [rust] # This enables `RUSTC_LOG=debug`, avoiding confusing situations @@ -17,3 +21,5 @@ debug-logging = true incremental = true # Print backtrace on internal compiler errors during bootstrap backtrace-on-ice = true +# Make the compiler and standard library faster to build, at the expense of a ~20% runtime slowdown. +lto = "off" diff --git a/src/bootstrap/defaults/config.user.toml b/src/bootstrap/defaults/config.user.toml index 48ae2fe44..25d9e649f 100644 --- a/src/bootstrap/defaults/config.user.toml +++ b/src/bootstrap/defaults/config.user.toml @@ -8,6 +8,12 @@ doc-stage = 2 # When compiling from source, you usually want all tools. extended = true +# Most users installing from source want to build all parts of the project from source. [llvm] -# Most users installing from source want to build all parts of the project from source, not just rustc itself. download-ci-llvm = false +[rust] +download-rustc = false + +[dist] +# Use better compression when preparing tarballs. +compression-profile = "balanced" diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9b2b54961..76aad16c1 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -18,14 +18,16 @@ use std::process::Command; use object::read::archive::ArchiveFile; use object::BinaryFormat; +use sha2::Digest; +use crate::bolt::{instrument_with_bolt, optimize_with_bolt}; use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; use crate::channel; use crate::compile; use crate::config::TargetSelection; use crate::doc::DocumentationFormat; -use crate::native; +use crate::llvm; use crate::tarball::{GeneratedTarball, OverlayKind, Tarball}; use crate::tool::{self, Tool}; use crate::util::{exe, is_dylib, output, t, timeit}; @@ -208,6 +210,8 @@ fn make_win_dist( rustc_dlls.push("libgcc_s_seh-1.dll"); } + // Libraries necessary to link the windows-gnu toolchains. + // System libraries will be preferred if they are available (see #67429). let target_libs = [ //MinGW libs "libgcc.a", @@ -221,6 +225,7 @@ fn make_win_dist( "libmoldname.a", "libpthread.a", //Windows import libs + //This should contain only the set of libraries necessary to link the standard library. "libadvapi32.a", "libbcrypt.a", "libcomctl32.a", @@ -234,6 +239,7 @@ fn make_win_dist( "libkernel32.a", "libmsimg32.a", "libmsvcrt.a", + "libntdll.a", "libodbc32.a", "libole32.a", "liboleaut32.a", @@ -322,7 +328,7 @@ impl Step for Mingw { /// without any extra installed software (e.g., we bundle gcc, libraries, etc). fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let host = self.host; - if !host.ends_with("pc-windows-gnu") { + if !host.ends_with("pc-windows-gnu") || !builder.config.dist_include_mingw_linker { return None; } @@ -378,7 +384,7 @@ impl Step for Rustc { // anything requiring us to distribute a license, but it's likely the // install will *also* include the rust-mingw package, which also needs // licenses, so to be safe we just include it here in all MinGW packages. - if host.ends_with("pc-windows-gnu") { + if host.ends_with("pc-windows-gnu") && builder.config.dist_include_mingw_linker { make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); } @@ -889,6 +895,8 @@ impl Step for Src { /// Creates the `rust-src` installer component fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + builder.update_submodule(&Path::new("src/llvm-project")); + let tarball = Tarball::new_targetless(builder, "rust-src"); // A lot of tools expect the rust-src component to be entirely in this directory, so if you @@ -965,9 +973,10 @@ impl Step for PlainSourceTarball { "RELEASES.md", "configure", "x.py", - "config.toml.example", + "config.example.toml", "Cargo.toml", "Cargo.lock", + ".gitmodules", ]; let src_dirs = ["src", "compiler", "library", "tests"]; @@ -1482,7 +1491,7 @@ impl Step for Extended { let xform = |p: &Path| { let mut contents = t!(fs::read_to_string(p)); - for tool in &["rust-demangler", "miri"] { + for tool in &["rust-demangler", "miri", "rust-docs"] { if !built_tools.contains(tool) { contents = filter(&contents, tool); } @@ -1579,11 +1588,10 @@ impl Step for Extended { prepare("rustc"); prepare("cargo"); prepare("rust-analysis"); - prepare("rust-docs"); prepare("rust-std"); prepare("clippy"); prepare("rust-analyzer"); - for tool in &["rust-demangler", "miri"] { + for tool in &["rust-docs", "rust-demangler", "miri"] { if built_tools.contains(tool) { prepare(tool); } @@ -1618,23 +1626,25 @@ impl Step for Extended { .arg("-out") .arg(exe.join("RustcGroup.wxs")), ); - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-docs") - .args(&heat_flags) - .arg("-cg") - .arg("DocsGroup") - .arg("-dr") - .arg("Docs") - .arg("-var") - .arg("var.DocsDir") - .arg("-out") - .arg(exe.join("DocsGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/squash-components.xsl")), - ); + if built_tools.contains("rust-docs") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-docs") + .args(&heat_flags) + .arg("-cg") + .arg("DocsGroup") + .arg("-dr") + .arg("Docs") + .arg("-var") + .arg("var.DocsDir") + .arg("-out") + .arg(exe.join("DocsGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/squash-components.xsl")), + ); + } builder.run( Command::new(&heat) .current_dir(&exe) @@ -1781,7 +1791,6 @@ impl Step for Extended { cmd.current_dir(&exe) .arg("-nologo") .arg("-dRustcDir=rustc") - .arg("-dDocsDir=rust-docs") .arg("-dCargoDir=cargo") .arg("-dStdDir=rust-std") .arg("-dAnalysisDir=rust-analysis") @@ -1793,6 +1802,9 @@ impl Step for Extended { .arg(&input); add_env(builder, &mut cmd, target); + if built_tools.contains("rust-docs") { + cmd.arg("-dDocsDir=rust-docs"); + } if built_tools.contains("rust-demangler") { cmd.arg("-dRustDemanglerDir=rust-demangler"); } @@ -1811,7 +1823,9 @@ impl Step for Extended { candle(&etc.join("msi/ui.wxs")); candle(&etc.join("msi/rustwelcomedlg.wxs")); candle("RustcGroup.wxs".as_ref()); - candle("DocsGroup.wxs".as_ref()); + if built_tools.contains("rust-docs") { + candle("DocsGroup.wxs".as_ref()); + } candle("CargoGroup.wxs".as_ref()); candle("StdGroup.wxs".as_ref()); candle("ClippyGroup.wxs".as_ref()); @@ -1848,7 +1862,6 @@ impl Step for Extended { .arg("ui.wixobj") .arg("rustwelcomedlg.wixobj") .arg("RustcGroup.wixobj") - .arg("DocsGroup.wixobj") .arg("CargoGroup.wixobj") .arg("StdGroup.wixobj") .arg("AnalysisGroup.wixobj") @@ -1864,6 +1877,9 @@ impl Step for Extended { if built_tools.contains("rust-demangler") { cmd.arg("RustDemanglerGroup.wixobj"); } + if built_tools.contains("rust-docs") { + cmd.arg("DocsGroup.wixobj"); + } if target.ends_with("windows-gnu") { cmd.arg("GccGroup.wixobj"); @@ -1904,6 +1920,26 @@ fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) { } } +fn install_llvm_file(builder: &Builder<'_>, source: &Path, destination: &Path) { + if builder.config.dry_run() { + return; + } + + // After LLVM is built, we modify (instrument or optimize) the libLLVM.so library file. + // This is not done in-place so that the built LLVM files are not "tainted" with BOLT. + // We perform the instrumentation/optimization here, on the fly, just before they are being + // packaged into some destination directory. + let postprocessed = if builder.config.llvm_bolt_profile_generate { + builder.ensure(BoltInstrument::new(source.to_path_buf())) + } else if let Some(path) = &builder.config.llvm_bolt_profile_use { + builder.ensure(BoltOptimize::new(source.to_path_buf(), path.into())) + } else { + source.to_path_buf() + }; + + builder.install(&postprocessed, destination, 0o644); +} + /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking. /// /// Returns whether the files were actually copied. @@ -1927,6 +1963,20 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir } } + // FIXME: for reasons I don't understand, the LLVM so in the `rustc` component is different than the one in `rust-dev`. + // Only the one in `rustc` works with the downloaded compiler. + if builder.download_rustc() && target == builder.build.build { + let src_libdir = builder.ci_rustc_dir(target).join("lib"); + for entry in t!(std::fs::read_dir(&src_libdir)) { + let entry = t!(entry); + if entry.file_name().to_str().unwrap().starts_with("libLLVM-") { + install_llvm_file(builder, &entry.path(), dst_libdir); + return !builder.config.dry_run(); + } + } + panic!("libLLVM.so not found in src_libdir {}!", src_libdir.display()); + } + // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely // clear why this is the case, though. llvm-config will emit the versioned @@ -1939,8 +1989,8 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir builder.install(&llvm_dylib_path, dst_libdir, 0o644); } !builder.config.dry_run() - } else if let Ok(native::LlvmResult { llvm_config, .. }) = - native::prebuilt_llvm_config(builder, target) + } else if let Ok(llvm::LlvmResult { llvm_config, .. }) = + llvm::prebuilt_llvm_config(builder, target) { let mut cmd = Command::new(llvm_config); cmd.arg("--libfiles"); @@ -1955,7 +2005,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir } else { PathBuf::from(file) }; - builder.install(&file, dst_libdir, 0o644); + install_llvm_file(builder, &file, dst_libdir); } !builder.config.dry_run() } else { @@ -1986,6 +2036,117 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection } } +/// Creates an output path to a BOLT-manipulated artifact for the given `file`. +/// The hash of the file is used to make sure that we don't mix BOLT artifacts amongst different +/// files with the same name. +/// +/// We need to keep the file-name the same though, to make sure that copying the manipulated file +/// to a directory will not change the final file path. +fn create_bolt_output_path(builder: &Builder<'_>, file: &Path, hash: &str) -> PathBuf { + let directory = builder.out.join("bolt").join(hash); + t!(fs::create_dir_all(&directory)); + directory.join(file.file_name().unwrap()) +} + +/// Instrument the provided file with BOLT. +/// Returns a path to the instrumented artifact. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BoltInstrument { + file: PathBuf, + hash: String, +} + +impl BoltInstrument { + fn new(file: PathBuf) -> Self { + let mut hasher = sha2::Sha256::new(); + hasher.update(t!(fs::read(&file))); + let hash = hex::encode(hasher.finalize().as_slice()); + + Self { file, hash } + } +} + +impl Step for BoltInstrument { + type Output = PathBuf; + + const ONLY_HOSTS: bool = false; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.build.config.dry_run() { + return self.file.clone(); + } + + if builder.build.config.llvm_from_ci { + println!("warning: trying to use BOLT with LLVM from CI, this will probably not work"); + } + + println!("Instrumenting {} with BOLT", self.file.display()); + + let output_path = create_bolt_output_path(builder, &self.file, &self.hash); + if !output_path.is_file() { + instrument_with_bolt(&self.file, &output_path); + } + output_path + } +} + +/// Optimize the provided file with BOLT. +/// Returns a path to the optimized artifact. +/// +/// The hash is stored in the step to make sure that we don't optimize the same file +/// twice (even under different file paths). +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BoltOptimize { + file: PathBuf, + profile: PathBuf, + hash: String, +} + +impl BoltOptimize { + fn new(file: PathBuf, profile: PathBuf) -> Self { + let mut hasher = sha2::Sha256::new(); + hasher.update(t!(fs::read(&file))); + hasher.update(t!(fs::read(&profile))); + let hash = hex::encode(hasher.finalize().as_slice()); + + Self { file, profile, hash } + } +} + +impl Step for BoltOptimize { + type Output = PathBuf; + + const ONLY_HOSTS: bool = false; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.build.config.dry_run() { + return self.file.clone(); + } + + if builder.build.config.llvm_from_ci { + println!("warning: trying to use BOLT with LLVM from CI, this will probably not work"); + } + + println!("Optimizing {} with BOLT", self.file.display()); + + let output_path = create_bolt_output_path(builder, &self.file, &self.hash); + if !output_path.is_file() { + optimize_with_bolt(&self.file, &self.profile, &output_path); + } + output_path + } +} + #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct LlvmTools { pub target: TargetSelection, @@ -2017,7 +2178,7 @@ impl Step for LlvmTools { } } - builder.ensure(crate::native::Llvm { target }); + builder.ensure(crate::llvm::Llvm { target }); let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); tarball.set_overlay(OverlayKind::LLVM); @@ -2076,10 +2237,10 @@ impl Step for RustDev { let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); tarball.set_overlay(OverlayKind::LLVM); - builder.ensure(crate::native::Llvm { target }); + builder.ensure(crate::llvm::Llvm { target }); // We want to package `lld` to use it with `download-ci-llvm`. - builder.ensure(crate::native::Lld { target }); + builder.ensure(crate::llvm::Lld { target }); let src_bindir = builder.llvm_out(target).join("bin"); // If updating this list, you likely want to change diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index cc80763ef..9ad98eb57 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -696,7 +696,7 @@ impl Step for Rustc { cargo.rustdocflag("-Znormalize-docs"); cargo.rustdocflag("--show-type-layout"); cargo.rustdocflag("--generate-link-to-definition"); - compile::rustc_cargo(builder, &mut cargo, target); + compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); cargo.arg("-Zunstable-options"); cargo.arg("-Zskip-rustdoc-fingerprint"); @@ -882,6 +882,7 @@ tool_doc!( // "cargo-credential-wincred", ] ); +tool_doc!(Tidy, "tidy", "src/tools/tidy", ["tidy"]); #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct ErrorIndex { @@ -1026,10 +1027,11 @@ impl Step for RustcBook { if self.validate { cmd.arg("--validate"); } - if !builder.unstable_features() { - // We need to validate nightly features, even on the stable channel. - cmd.env("RUSTC_BOOTSTRAP", "1"); - } + // We need to validate nightly features, even on the stable channel. + // Set this unconditionally as the stage0 compiler may be being used to + // document. + cmd.env("RUSTC_BOOTSTRAP", "1"); + // If the lib directories are in an unusual location (changed in // config.toml), then this needs to explicitly update the dylib search // path. diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp index 94630e40f..36f9aaa59 100644 --- a/src/bootstrap/download-ci-llvm-stamp +++ b/src/bootstrap/download-ci-llvm-stamp @@ -1,4 +1,4 @@ Change this file to make users of the `download-ci-llvm` configuration download a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. -Last change is for: https://github.com/rust-lang/rust/pull/104748 +Last change is for: https://github.com/rust-lang/rust/pull/109373 diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index d1e2149d3..242515565 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -12,7 +12,7 @@ use xz2::bufread::XzDecoder; use crate::{ config::RustfmtMetadata, - native::detect_llvm_sha, + llvm::detect_llvm_sha, t, util::{check_run, exe, program_out_of_date, try_run}, Config, @@ -367,26 +367,70 @@ impl Config { pub(crate) fn download_ci_rustc(&self, commit: &str) { self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); + let version = self.artifact_version_part(commit); + // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the + // `rustc_private` crates for tools. + let extra_components = ["rustc-dev"]; + + self.download_toolchain( + &version, + "ci-rustc", + commit, + &extra_components, + Self::download_ci_component, + ); + } + + pub(crate) fn download_beta_toolchain(&self) { + self.verbose(&format!("downloading stage0 beta artifacts")); + + let date = &self.stage0_metadata.compiler.date; + let version = &self.stage0_metadata.compiler.version; + let extra_components = ["cargo"]; + + let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { + config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") + }; + + self.download_toolchain( + version, + "stage0", + date, + &extra_components, + download_beta_component, + ); + } + + fn download_toolchain( + &self, + // FIXME(ozkanonur) use CompilerMetadata instead of `version: &str` + version: &str, + sysroot: &str, + stamp_key: &str, + extra_components: &[&str], + download_component: fn(&Config, String, &str, &str), + ) { let host = self.build.triple; - let bin_root = self.out.join(host).join("ci-rustc"); + let bin_root = self.out.join(host).join(sysroot); let rustc_stamp = bin_root.join(".rustc-stamp"); - if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) + if !bin_root.join("bin").join(exe("rustc", self.build)).exists() + || program_out_of_date(&rustc_stamp, stamp_key) { if bin_root.exists() { t!(fs::remove_dir_all(&bin_root)); } let filename = format!("rust-std-{version}-{host}.tar.xz"); let pattern = format!("rust-std-{host}"); - self.download_ci_component(filename, &pattern, commit); + download_component(self, filename, &pattern, stamp_key); let filename = format!("rustc-{version}-{host}.tar.xz"); - self.download_ci_component(filename, "rustc", commit); - // download-rustc doesn't need its own cargo, it can just use beta's. - let filename = format!("rustc-dev-{version}-{host}.tar.xz"); - self.download_ci_component(filename, "rustc-dev", commit); - let filename = format!("rust-src-{version}.tar.xz"); - self.download_ci_component(filename, "rust-src", commit); + download_component(self, filename, "rustc", stamp_key); + + for component in extra_components { + let filename = format!("{component}-{version}-{host}.tar.xz"); + download_component(self, filename, component, stamp_key); + } if self.should_fix_bins_and_dylibs() { self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); @@ -403,7 +447,7 @@ impl Config { } } - t!(fs::write(rustc_stamp, commit)); + t!(fs::write(rustc_stamp, stamp_key)); } } diff --git a/src/bootstrap/dylib_util.rs b/src/bootstrap/dylib_util.rs index 6d75272c5..b14c0bed6 100644 --- a/src/bootstrap/dylib_util.rs +++ b/src/bootstrap/dylib_util.rs @@ -12,6 +12,8 @@ pub fn dylib_path_var() -> &'static str { "DYLD_LIBRARY_PATH" } else if cfg!(target_os = "haiku") { "LIBRARY_PATH" + } else if cfg!(target_os = "aix") { + "LIBPATH" } else { "LD_LIBRARY_PATH" } diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 9d1504c34..b6f5f3103 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -67,8 +67,6 @@ pub struct Flags { // true => deny, false => warn pub deny_warnings: Option<bool>, - pub llvm_skip_rebuild: Option<bool>, - pub rust_profile_use: Option<String>, pub rust_profile_generate: Option<String>, @@ -86,8 +84,7 @@ pub struct Flags { pub free_args: Option<Vec<String>>, } -#[derive(Debug)] -#[cfg_attr(test, derive(Clone))] +#[derive(Debug, Clone)] pub enum Subcommand { Build { paths: Vec<PathBuf>, @@ -151,6 +148,9 @@ pub enum Subcommand { Setup { profile: Option<PathBuf>, }, + Suggest { + run: bool, + }, } impl Default for Subcommand { @@ -185,6 +185,7 @@ Subcommands: install Install distribution artifacts run, r Run tools contained in this repository setup Create a config.toml (making it easier to use `x.py` itself) + suggest Suggest a subset of tests to run, based on modified files To learn more about a subcommand, run `./x.py <subcommand> -h`", ); @@ -251,14 +252,6 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", opts.optopt("", "color", "whether to use color in cargo and rustc output", "STYLE"); opts.optopt( "", - "llvm-skip-rebuild", - "whether rebuilding llvm should be skipped \ - a VALUE of TRUE indicates that llvm will not be rebuilt \ - VALUE overrides the skip-rebuild option in config.toml.", - "VALUE", - ); - opts.optopt( - "", "rust-profile-generate", "generate PGO profile with rustc build", "PROFILE", @@ -359,6 +352,9 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", Kind::Run => { opts.optmulti("", "args", "arguments for the tool", "ARGS"); } + Kind::Suggest => { + opts.optflag("", "run", "run suggested tests"); + } _ => {} }; @@ -575,7 +571,7 @@ Arguments: Profile::all_for_help(" ").trim_end() )); } - Kind::Bench | Kind::Clean | Kind::Dist | Kind::Install => {} + Kind::Bench | Kind::Clean | Kind::Dist | Kind::Install | Kind::Suggest => {} }; // Get any optional paths which occur after the subcommand let mut paths = matches.free[1..].iter().map(|p| p.into()).collect::<Vec<PathBuf>>(); @@ -636,6 +632,7 @@ Arguments: Kind::Format => Subcommand::Format { check: matches.opt_present("check"), paths }, Kind::Dist => Subcommand::Dist { paths }, Kind::Install => Subcommand::Install { paths }, + Kind::Suggest => Subcommand::Suggest { run: matches.opt_present("run") }, Kind::Run => { if paths.is_empty() { println!("\nrun requires at least a path!\n"); @@ -714,9 +711,6 @@ Arguments: .collect::<Vec<_>>(), include_default_paths: matches.opt_present("include-default-paths"), deny_warnings: parse_deny_warnings(&matches), - llvm_skip_rebuild: matches.opt_str("llvm-skip-rebuild").map(|s| s.to_lowercase()).map( - |s| s.parse::<bool>().expect("`llvm-skip-rebuild` should be either true or false"), - ), color: matches .opt_get_default("color", Color::Auto) .expect("`color` should be `always`, `never`, or `auto`"), @@ -747,6 +741,7 @@ impl Subcommand { Subcommand::Install { .. } => Kind::Install, Subcommand::Run { .. } => Kind::Run, Subcommand::Setup { .. } => Kind::Setup, + Subcommand::Suggest { .. } => Kind::Suggest, } } diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs index 6d5753e8a..b79969663 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/format.rs @@ -2,6 +2,7 @@ use crate::builder::Builder; use crate::util::{output, program_out_of_date, t}; +use build_helper::ci::CiEnv; use build_helper::git::get_git_modified_files; use ignore::WalkBuilder; use std::collections::VecDeque; @@ -144,8 +145,10 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { let untracked_paths = untracked_paths_output .lines() .filter(|entry| entry.starts_with("??")) - .map(|entry| { - entry.split(' ').nth(1).expect("every git status entry should list a path") + .filter_map(|entry| { + let path = + entry.split(' ').nth(1).expect("every git status entry should list a path"); + path.ends_with(".rs").then_some(path) }); for untracked_path in untracked_paths { println!("skip untracked path {} during rustfmt invocations", untracked_path); @@ -156,11 +159,20 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { // preventing the latter from being formatted. ignore_fmt.add(&format!("!/{}", untracked_path)).expect(&untracked_path); } - if !check && paths.is_empty() { + // Only check modified files locally to speed up runtime. + // We still check all files in CI to avoid bugs in `get_modified_rs_files` letting regressions slip through; + // we also care about CI time less since this is still very fast compared to building the compiler. + if !CiEnv::is_ci() && paths.is_empty() { match get_modified_rs_files(build) { Ok(Some(files)) => { + if files.len() <= 10 { + for file in &files { + println!("formatting modified file {file}"); + } + } else { + println!("formatting {} modified files", files.len()); + } for file in files { - println!("formatting modified file {file}"); ignore_fmt.add(&format!("/{file}")).expect(&file); } } diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index ac3843c33..42d895a34 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -210,10 +210,13 @@ install!((self, builder, _config), } }; LlvmTools, alias = "llvm-tools", Self::should_build(_config), only_hosts: true, { - let tarball = builder - .ensure(dist::LlvmTools { target: self.target }) - .expect("missing llvm-tools"); - install_sh(builder, "llvm-tools", self.compiler.stage, Some(self.target), &tarball); + if let Some(tarball) = builder.ensure(dist::LlvmTools { target: self.target }) { + install_sh(builder, "llvm-tools", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping llvm-tools stage{} ({}): external LLVM", self.compiler.stage, self.target), + ); + } }; Rustfmt, alias = "rustfmt", Self::should_build(_config), only_hosts: true, { if let Some(tarball) = builder.ensure(dist::Rustfmt { diff --git a/src/bootstrap/job.rs b/src/bootstrap/job.rs index 5c0322e18..4fb00f65d 100644 --- a/src/bootstrap/job.rs +++ b/src/bootstrap/job.rs @@ -27,52 +27,54 @@ //! Note that this module has a #[cfg(windows)] above it as none of this logic //! is required on Unix. -#![allow(nonstandard_style, dead_code)] - use crate::Build; use std::env; +use std::ffi::c_void; use std::io; use std::mem; -use std::ptr; -use winapi::shared::minwindef::{DWORD, FALSE, LPVOID}; -use winapi::um::errhandlingapi::SetErrorMode; -use winapi::um::handleapi::{CloseHandle, DuplicateHandle}; -use winapi::um::jobapi2::{AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject}; -use winapi::um::processthreadsapi::{GetCurrentProcess, OpenProcess}; -use winapi::um::winbase::{BELOW_NORMAL_PRIORITY_CLASS, SEM_NOGPFAULTERRORBOX}; -use winapi::um::winnt::{ - JobObjectExtendedLimitInformation, DUPLICATE_SAME_ACCESS, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, - JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, PROCESS_DUP_HANDLE, +use windows::{ + core::PCWSTR, + Win32::Foundation::{CloseHandle, DuplicateHandle, DUPLICATE_SAME_ACCESS, HANDLE}, + Win32::System::Diagnostics::Debug::{SetErrorMode, SEM_NOGPFAULTERRORBOX, THREAD_ERROR_MODE}, + Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, + SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, + }, + Win32::System::Threading::{ + GetCurrentProcess, OpenProcess, BELOW_NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, + }, }; pub unsafe fn setup(build: &mut Build) { // Enable the Windows Error Reporting dialog which msys disables, // so we can JIT debug rustc - let mode = SetErrorMode(0); + let mode = SetErrorMode(THREAD_ERROR_MODE::default()); + let mode = THREAD_ERROR_MODE(mode); SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); // Create a new job object for us to use - let job = CreateJobObjectW(ptr::null_mut(), ptr::null()); - assert!(!job.is_null(), "{}", io::Error::last_os_error()); + let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); // Indicate that when all handles to the job object are gone that all // process in the object should be killed. Note that this includes our // entire process tree by default because we've added ourselves and our // children will reside in the job by default. - let mut info = mem::zeroed::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>(); + let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; if build.config.low_priority { info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; - info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS; + info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; } let r = SetInformationJobObject( job, JobObjectExtendedLimitInformation, - &mut info as *mut _ as LPVOID, - mem::size_of_val(&info) as DWORD, - ); - assert!(r != 0, "{}", io::Error::last_os_error()); + &info as *const _ as *const c_void, + mem::size_of_val(&info) as u32, + ) + .ok(); + assert!(r.is_ok(), "{}", io::Error::last_os_error()); // Assign our process to this job object. Note that if this fails, one very // likely reason is that we are ourselves already in a job object! This can @@ -83,8 +85,8 @@ pub unsafe fn setup(build: &mut Build) { // Also note that nested jobs (why this might fail) are supported in recent // versions of Windows, but the version of Windows that our bots are running // at least don't support nested job objects. - let r = AssignProcessToJobObject(job, GetCurrentProcess()); - if r == 0 { + let r = AssignProcessToJobObject(job, GetCurrentProcess()).ok(); + if r.is_err() { CloseHandle(job); return; } @@ -102,31 +104,32 @@ pub unsafe fn setup(build: &mut Build) { Err(..) => return, }; - let parent = OpenProcess(PROCESS_DUP_HANDLE, FALSE, pid.parse().unwrap()); - - // If we get a null parent pointer here, it is possible that either - // we have got an invalid pid or the parent process has been closed. - // Since the first case rarely happens - // (only when wrongly setting the environmental variable), - // so it might be better to improve the experience of the second case - // when users have interrupted the parent process and we don't finish - // duplicating the handle yet. - // We just need close the job object if that occurs. - if parent.is_null() { - CloseHandle(job); - return; - } + let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { + Some(parent) => parent, + _ => { + // If we get a null parent pointer here, it is possible that either + // we have an invalid pid or the parent process has been closed. + // Since the first case rarely happens + // (only when wrongly setting the environmental variable), + // it might be better to improve the experience of the second case + // when users have interrupted the parent process and we haven't finish + // duplicating the handle yet. We just need close the job object if that occurs. + CloseHandle(job); + return; + } + }; - let mut parent_handle = ptr::null_mut(); + let mut parent_handle = HANDLE::default(); let r = DuplicateHandle( GetCurrentProcess(), job, parent, &mut parent_handle, 0, - FALSE, + false, DUPLICATE_SAME_ACCESS, - ); + ) + .ok(); // If this failed, well at least we tried! An example of DuplicateHandle // failing in the past has been when the wrong python2 package spawned this @@ -134,7 +137,7 @@ pub unsafe fn setup(build: &mut Build) { // `mingw-w64-x86_64-python2`. Not sure why it failed, but the "failure // mode" here is that we only clean everything up when the build system // dies, not when the python parent does, so not too bad. - if r != 0 { + if r.is_err() { CloseHandle(job); } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 950f3b151..419bcbc63 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -21,7 +21,6 @@ use std::collections::{HashMap, HashSet}; use std::env; use std::fs::{self, File}; use std::io; -use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::str; @@ -53,11 +52,13 @@ mod download; mod flags; mod format; mod install; +mod llvm; mod metadata; -mod native; +mod render_tests; mod run; mod sanity; mod setup; +mod suggest; mod tarball; mod test; mod tool; @@ -88,6 +89,7 @@ pub use crate::builder::PathSet; use crate::cache::{Interned, INTERNER}; pub use crate::config::Config; pub use crate::flags::Subcommand; +use termcolor::{ColorChoice, StandardStream, WriteColor}; const LLVM_TOOLS: &[&str] = &[ "llvm-cov", // used to generate coverage report @@ -123,11 +125,13 @@ const EXTRA_CHECK_CFGS: &[(Option<Mode>, &'static str, Option<&[&'static str]>)] (Some(Mode::Std), "no_rc", None), (Some(Mode::Std), "no_sync", None), (Some(Mode::Std), "freebsd12", None), + (Some(Mode::Std), "freebsd13", None), (Some(Mode::Std), "backtrace_in_libstd", None), /* Extra values not defined in the built-in targets yet, but used in std */ (Some(Mode::Std), "target_env", Some(&["libnx"])), // (Some(Mode::Std), "target_os", Some(&[])), - (Some(Mode::Std), "target_arch", Some(&["asmjs", "spirv", "nvptx", "xtensa"])), + // #[cfg(bootstrap)] loongarch64 + (Some(Mode::Std), "target_arch", Some(&["asmjs", "spirv", "nvptx", "xtensa", "loongarch64"])), /* Extra names used by dependencies */ // FIXME: Used by serde_json, but we should not be triggering on external dependencies. (Some(Mode::Rustc), "no_btreemap_remove_entry", None), @@ -144,6 +148,11 @@ const EXTRA_CHECK_CFGS: &[(Option<Mode>, &'static str, Option<&[&'static str]>)] // FIXME: Used by filetime, but we should not be triggering on external dependencies. (Some(Mode::Rustc), "emulate_second_only_system", None), (Some(Mode::ToolRustc), "emulate_second_only_system", None), + // Needed to avoid the need to copy windows.lib into the sysroot. + (Some(Mode::Rustc), "windows_raw_dylib", None), + (Some(Mode::ToolRustc), "windows_raw_dylib", None), + // #[cfg(bootstrap)] ohos + (Some(Mode::Std), "target_env", Some(&["ohos"])), ]; /// A structure representing a Rust compiler. @@ -182,6 +191,7 @@ pub enum GitRepo { /// although most functions are implemented as free functions rather than /// methods specifically on this structure itself (to make it easier to /// organize). +#[cfg_attr(not(feature = "build-metrics"), derive(Clone))] pub struct Build { /// User-specified configuration from `config.toml`. config: Config, @@ -235,7 +245,7 @@ pub struct Build { metrics: metrics::BuildMetrics, } -#[derive(Debug)] +#[derive(Debug, Clone)] struct Crate { name: Interned<String>, deps: HashSet<Interned<String>>, @@ -351,14 +361,14 @@ impl Build { #[cfg(not(unix))] let is_sudo = false; - let ignore_git = config.ignore_git; - let rust_info = channel::GitInfo::new(ignore_git, &src); - let cargo_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/cargo")); + let omit_git_hash = config.omit_git_hash; + let rust_info = channel::GitInfo::new(omit_git_hash, &src); + let cargo_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); let rust_analyzer_info = - channel::GitInfo::new(ignore_git, &src.join("src/tools/rust-analyzer")); - let clippy_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/clippy")); - let miri_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/miri")); - let rustfmt_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/rustfmt")); + channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); + let clippy_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); + let miri_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); + let rustfmt_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); // we always try to use git for LLVM builds let in_tree_llvm_info = channel::GitInfo::new(false, &src.join("src/llvm-project")); @@ -482,12 +492,7 @@ impl Build { // Make sure we update these before gathering metadata so we don't get an error about missing // Cargo.toml files. - let rust_submodules = [ - "src/tools/rust-installer", - "src/tools/cargo", - "library/backtrace", - "library/stdarch", - ]; + let rust_submodules = ["src/tools/cargo", "library/backtrace", "library/stdarch"]; for s in rust_submodules { build.update_submodule(Path::new(s)); } @@ -502,16 +507,18 @@ impl Build { let build_triple = build.out.join(&build.build.triple); t!(fs::create_dir_all(&build_triple)); let host = build.out.join("host"); - if let Err(e) = symlink_dir(&build.config, &build_triple, &host) { - if e.kind() != ErrorKind::AlreadyExists { - panic!( - "symlink_dir({} => {}) failed with {}", - host.display(), - build_triple.display(), - e - ); - } - } + if host.is_symlink() { + // Left over from a previous build; overwrite it. + // This matters if `build.build` has changed between invocations. + #[cfg(windows)] + t!(fs::remove_dir(&host)); + #[cfg(not(windows))] + t!(fs::remove_file(&host)); + } + t!( + symlink_dir(&build.config, &build_triple, &host), + format!("symlink_dir({} => {}) failed", host.display(), build_triple.display()) + ); build } @@ -652,13 +659,20 @@ impl Build { job::setup(self); } - if let Subcommand::Format { check, paths } = &self.config.cmd { - return format::format(&builder::Builder::new(&self), *check, &paths); - } - // Download rustfmt early so that it can be used in rust-analyzer configs. let _ = &builder::Builder::new(&self).initial_rustfmt(); + // hardcoded subcommands + match &self.config.cmd { + Subcommand::Format { check, paths } => { + return format::format(&builder::Builder::new(&self), *check, &paths); + } + Subcommand::Suggest { run } => { + return suggest::suggest(&builder::Builder::new(&self), *run); + } + _ => (), + } + { let builder = builder::Builder::new(&self); if let Some(path) = builder.paths.get(0) { @@ -800,6 +814,11 @@ impl Build { self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir()) } + /// Directory where the extracted `rustc-dev` component is stored. + fn ci_rustc_dir(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("ci-rustc") + } + /// Root output directory for LLVM compiled for `target` /// /// Note that if LLVM is configured externally then the directory returned @@ -1204,12 +1223,22 @@ impl Build { /// /// When all of these conditions are met the build will lift artifacts from /// the previous stage forward. - fn force_use_stage1(&self, compiler: Compiler, target: TargetSelection) -> bool { + fn force_use_stage1(&self, stage: u32, target: TargetSelection) -> bool { !self.config.full_bootstrap - && compiler.stage >= 2 + && !self.config.download_rustc() + && stage >= 2 && (self.hosts.iter().any(|h| *h == target) || target == self.build) } + /// Checks whether the `compiler` compiling for `target` should be forced to + /// use a stage2 compiler instead. + /// + /// When we download the pre-compiled version of rustc and compiler stage is >= 2, + /// it should be forced to use a stage2 compiler. + fn force_use_stage2(&self, stage: u32) -> bool { + self.config.download_rustc() && stage >= 2 + } + /// Given `num` in the form "a.b.c" return a "release string" which /// describes the release version number. /// @@ -1219,7 +1248,7 @@ impl Build { match &self.config.channel[..] { "stable" => num.to_string(), "beta" => { - if self.rust_info().is_managed_git_subrepository() && !self.config.ignore_git { + if self.rust_info().is_managed_git_subrepository() && !self.config.omit_git_hash { format!("{}-beta.{}", num, self.beta_prerelease_version()) } else { format!("{}-beta", num) @@ -1575,6 +1604,31 @@ to download LLVM rather than building it. self.config.ninja_in_file } + + pub fn colored_stdout<R, F: FnOnce(&mut dyn WriteColor) -> R>(&self, f: F) -> R { + self.colored_stream_inner(StandardStream::stdout, self.config.stdout_is_tty, f) + } + + pub fn colored_stderr<R, F: FnOnce(&mut dyn WriteColor) -> R>(&self, f: F) -> R { + self.colored_stream_inner(StandardStream::stderr, self.config.stderr_is_tty, f) + } + + fn colored_stream_inner<R, F, C>(&self, constructor: C, is_tty: bool, f: F) -> R + where + C: Fn(ColorChoice) -> StandardStream, + F: FnOnce(&mut dyn WriteColor) -> R, + { + let choice = match self.config.color { + flags::Color::Always => ColorChoice::Always, + flags::Color::Never => ColorChoice::Never, + flags::Color::Auto if !is_tty => ColorChoice::Never, + flags::Color::Auto => ColorChoice::Auto, + }; + let mut stream = constructor(choice); + let result = f(&mut stream); + stream.reset().unwrap(); + result + } } #[cfg(unix)] diff --git a/src/bootstrap/native.rs b/src/bootstrap/llvm.rs index 5987b641b..a893c3a47 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/llvm.rs @@ -16,7 +16,6 @@ use std::io; use std::path::{Path, PathBuf}; use std::process::Command; -use crate::bolt::{instrument_with_bolt_inplace, optimize_library_with_bolt_inplace}; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::channel; use crate::config::{Config, TargetSelection}; @@ -109,15 +108,6 @@ pub fn prebuilt_llvm_config( let stamp = out_dir.join("llvm-finished-building"); let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); - if builder.config.llvm_skip_rebuild && stamp.path.exists() { - builder.info( - "Warning: \ - Using a potentially stale build of LLVM; \ - This may not behave well.", - ); - return Ok(res); - } - if stamp.is_done() { if stamp.hash.is_none() { builder.info( @@ -226,7 +216,7 @@ pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { /// Returns true if we're running in CI with modified LLVM (and thus can't download it) pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool { - CiEnv::is_ci() && { + CiEnv::is_ci() && config.rust_info.is_managed_git_subrepository() && { // We assume we have access to git, so it's okay to unconditionally pass // `true` here. let llvm_sha = detect_llvm_sha(config, true); @@ -296,12 +286,12 @@ impl Step for Llvm { (true, true) => "RelWithDebInfo", }; - // NOTE: remember to also update `config.toml.example` when changing the + // NOTE: remember to also update `config.example.toml` when changing the // defaults! let llvm_targets = match &builder.config.llvm_targets { Some(s) => s, None => { - "AArch64;ARM;BPF;Hexagon;MSP430;Mips;NVPTX;PowerPC;RISCV;\ + "AArch64;ARM;BPF;Hexagon;LoongArch;MSP430;Mips;NVPTX;PowerPC;RISCV;\ Sparc;SystemZ;WebAssembly;X86" } }; @@ -314,10 +304,12 @@ impl Step for Llvm { let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" }; let plugins = if builder.config.llvm_plugins { "ON" } else { "OFF" }; let enable_tests = if builder.config.llvm_tests { "ON" } else { "OFF" }; + let enable_warnings = if builder.config.llvm_enable_warnings { "ON" } else { "OFF" }; cfg.out_dir(&out_dir) .profile(profile) .define("LLVM_ENABLE_ASSERTIONS", assertions) + .define("LLVM_UNREACHABLE_OPTIMIZE", "OFF") .define("LLVM_ENABLE_PLUGINS", plugins) .define("LLVM_TARGETS_TO_BUILD", llvm_targets) .define("LLVM_EXPERIMENTAL_TARGETS_TO_BUILD", llvm_exp_targets) @@ -331,7 +323,8 @@ impl Step for Llvm { .define("LLVM_ENABLE_Z3_SOLVER", "OFF") .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string()) .define("LLVM_TARGET_ARCH", target_native.split('-').next().unwrap()) - .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native); + .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native) + .define("LLVM_ENABLE_WARNINGS", enable_warnings); // Parts of our test suite rely on the `FileCheck` tool, which is built by default in // `build/$TARGET/llvm/build/bin` is but *not* then installed to `build/$TARGET/llvm/bin`. @@ -441,11 +434,6 @@ impl Step for Llvm { } } - // Workaround for ppc32 lld limitation - if target == "powerpc-unknown-freebsd" { - ldflags.exe.push(" -fuse-ld=bfd"); - } - // https://llvm.org/docs/HowToCrossCompileLLVM.html if target != builder.config.build { let LlvmResult { llvm_config, .. } = @@ -493,11 +481,6 @@ impl Step for Llvm { cfg.define(key, val); } - // FIXME: we don't actually need to build all LLVM tools and all LLVM - // libraries here, e.g., we just want a few components and a few - // tools. Figure out how to filter them down and only build the right - // tools and libs on all platforms. - if builder.config.dry_run() { return res; } @@ -523,39 +506,13 @@ impl Step for Llvm { } } - // After LLVM is built, we modify (instrument or optimize) the libLLVM.so library file - // in place. This is fine, because currently we do not support incrementally rebuilding - // LLVM after a configuration change, so to rebuild it the build files have to be removed, - // which will also remove these modified files. - if builder.config.llvm_bolt_profile_generate { - instrument_with_bolt_inplace(&get_built_llvm_lib_path(&res.llvm_config)); - } - if let Some(path) = &builder.config.llvm_bolt_profile_use { - optimize_library_with_bolt_inplace( - &get_built_llvm_lib_path(&res.llvm_config), - &Path::new(path), - ); - } - t!(stamp.write()); res } } -/// Returns path to a built LLVM library (libLLVM.so). -/// Assumes that we have built LLVM into a single library file. -fn get_built_llvm_lib_path(llvm_config_path: &Path) -> PathBuf { - let mut cmd = Command::new(llvm_config_path); - cmd.arg("--libfiles"); - PathBuf::from(output(&mut cmd).trim()) -} - fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) { - if !builder.config.llvm_version_check { - return; - } - if builder.config.dry_run() { return; } @@ -606,6 +563,8 @@ fn configure_cmake( cfg.define("CMAKE_SYSTEM_NAME", "Haiku"); } else if target.contains("solaris") || target.contains("illumos") { cfg.define("CMAKE_SYSTEM_NAME", "SunOS"); + } else if target.contains("linux") { + cfg.define("CMAKE_SYSTEM_NAME", "Linux"); } // When cross-compiling we should also set CMAKE_SYSTEM_VERSION, but in // that case like CMake we cannot easily determine system version either. @@ -906,71 +865,6 @@ impl Step for Lld { } #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct TestHelpers { - pub target: TargetSelection, -} - -impl Step for TestHelpers { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("tests/auxiliary/rust_test_helpers.c") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(TestHelpers { target: run.target }) - } - - /// Compiles the `rust_test_helpers.c` library which we used in various - /// `run-pass` tests for ABI testing. - fn run(self, builder: &Builder<'_>) { - if builder.config.dry_run() { - return; - } - // The x86_64-fortanix-unknown-sgx target doesn't have a working C - // toolchain. However, some x86_64 ELF objects can be linked - // without issues. Use this hack to compile the test helpers. - let target = if self.target == "x86_64-fortanix-unknown-sgx" { - TargetSelection::from_user("x86_64-unknown-linux-gnu") - } else { - self.target - }; - let dst = builder.test_helpers_out(target); - let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); - if up_to_date(&src, &dst.join("librust_test_helpers.a")) { - return; - } - - builder.info("Building test helpers"); - t!(fs::create_dir_all(&dst)); - let mut cfg = cc::Build::new(); - // FIXME: Workaround for https://github.com/emscripten-core/emscripten/issues/9013 - if target.contains("emscripten") { - cfg.pic(false); - } - - // We may have found various cross-compilers a little differently due to our - // extra configuration, so inform cc of these compilers. Note, though, that - // on MSVC we still need cc's detection of env vars (ugh). - if !target.contains("msvc") { - if let Some(ar) = builder.ar(target) { - cfg.archiver(ar); - } - cfg.compiler(builder.cc(target)); - } - cfg.cargo_metadata(false) - .out_dir(&dst) - .target(&target.triple) - .host(&builder.config.build.triple) - .opt_level(0) - .warnings(false) - .debug(false) - .file(builder.src.join("tests/auxiliary/rust_test_helpers.c")) - .compile("rust_test_helpers"); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct Sanitizers { pub target: TargetSelection, } @@ -1109,6 +1003,9 @@ fn supported_sanitizers( "aarch64-unknown-linux-gnu" => { common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) } + "aarch64-unknown-linux-ohos" => { + common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) + } "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), "x86_64-unknown-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), "x86_64-apple-ios" => darwin_libs("iossim", &["asan", "tsan"]), @@ -1190,6 +1087,8 @@ impl Step for CrtBeginEnd { /// Build crtbegin.o/crtend.o for musl target. fn run(self, builder: &Builder<'_>) -> Self::Output { + builder.update_submodule(&Path::new("src/llvm-project")); + let out_dir = builder.native_dir(self.target).join("crt"); if builder.config.dry_run() { @@ -1256,6 +1155,8 @@ impl Step for Libunwind { /// Build linunwind.a fn run(self, builder: &Builder<'_>) -> Self::Output { + builder.update_submodule(&Path::new("src/llvm-project")); + if builder.config.dry_run() { return PathBuf::new(); } diff --git a/src/bootstrap/metrics.rs b/src/bootstrap/metrics.rs index 2e62c9507..82b123ec8 100644 --- a/src/bootstrap/metrics.rs +++ b/src/bootstrap/metrics.rs @@ -11,7 +11,7 @@ use serde_derive::{Deserialize, Serialize}; use std::cell::RefCell; use std::fs::File; use std::io::BufWriter; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime}; use sysinfo::{CpuExt, System, SystemExt}; pub(crate) struct BuildMetrics { @@ -27,6 +27,7 @@ impl BuildMetrics { system_info: System::new(), timer_start: None, invocation_timer_start: Instant::now(), + invocation_start: SystemTime::now(), }); BuildMetrics { state } @@ -51,6 +52,7 @@ impl BuildMetrics { duration_excluding_children_sec: Duration::ZERO, children: Vec::new(), + tests: Vec::new(), }); } @@ -72,6 +74,16 @@ impl BuildMetrics { } } + pub(crate) fn record_test(&self, name: &str, outcome: TestOutcome) { + let mut state = self.state.borrow_mut(); + state + .running_steps + .last_mut() + .unwrap() + .tests + .push(Test { name: name.to_string(), outcome }); + } + fn collect_stats(&self, state: &mut MetricsState) { let step = state.running_steps.last_mut().unwrap(); @@ -113,6 +125,11 @@ impl BuildMetrics { } }; invocations.push(JsonInvocation { + start_time: state + .invocation_start + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(), children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(), }); @@ -125,6 +142,14 @@ impl BuildMetrics { } fn prepare_json_step(&self, step: StepMetrics) -> JsonNode { + let mut children = Vec::new(); + children.extend(step.children.into_iter().map(|child| self.prepare_json_step(child))); + children.extend( + step.tests + .into_iter() + .map(|test| JsonNode::Test { name: test.name, outcome: test.outcome }), + ); + JsonNode::RustbuildStep { type_: step.type_, debug_repr: step.debug_repr, @@ -135,11 +160,7 @@ impl BuildMetrics { / step.duration_excluding_children_sec.as_secs_f64(), }, - children: step - .children - .into_iter() - .map(|child| self.prepare_json_step(child)) - .collect(), + children, } } } @@ -151,6 +172,7 @@ struct MetricsState { system_info: System, timer_start: Option<Instant>, invocation_timer_start: Instant, + invocation_start: SystemTime, } struct StepMetrics { @@ -161,6 +183,12 @@ struct StepMetrics { duration_excluding_children_sec: Duration, children: Vec<StepMetrics>, + tests: Vec<Test>, +} + +struct Test { + name: String, + outcome: TestOutcome, } #[derive(Serialize, Deserialize)] @@ -173,6 +201,10 @@ struct JsonRoot { #[derive(Serialize, Deserialize)] #[serde(rename_all = "snake_case")] struct JsonInvocation { + // Unix timestamp in seconds + // + // This is necessary to easily correlate this invocation with logs or other data. + start_time: u64, duration_including_children_sec: f64, children: Vec<JsonNode>, } @@ -190,6 +222,19 @@ enum JsonNode { children: Vec<JsonNode>, }, + Test { + name: String, + #[serde(flatten)] + outcome: TestOutcome, + }, +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub(crate) enum TestOutcome { + Passed, + Failed, + Ignored { ignore_reason: Option<String> }, } #[derive(Serialize, Deserialize)] diff --git a/src/bootstrap/render_tests.rs b/src/bootstrap/render_tests.rs new file mode 100644 index 000000000..19019ad2c --- /dev/null +++ b/src/bootstrap/render_tests.rs @@ -0,0 +1,371 @@ +//! This module renders the JSON output of libtest into a human-readable form, trying to be as +//! similar to libtest's native output as possible. +//! +//! This is needed because we need to use libtest in JSON mode to extract granluar information +//! about the executed tests. Doing so suppresses the human-readable output, and (compared to Cargo +//! and rustc) libtest doesn't include the rendered human-readable output as a JSON field. We had +//! to reimplement all the rendering logic in this module because of that. + +use crate::builder::Builder; +use std::io::{BufRead, BufReader, Write}; +use std::process::{ChildStdout, Command, Stdio}; +use std::time::Duration; +use termcolor::{Color, ColorSpec, WriteColor}; + +const TERSE_TESTS_PER_LINE: usize = 88; + +pub(crate) fn add_flags_and_try_run_tests(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if cmd.get_args().position(|arg| arg == "--").is_none() { + cmd.arg("--"); + } + cmd.args(&["-Z", "unstable-options", "--format", "json"]); + + try_run_tests(builder, cmd) +} + +pub(crate) fn try_run_tests(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if builder.config.dry_run() { + return true; + } + + if !run_tests(builder, cmd) { + if builder.fail_fast { + crate::detail_exit(1); + } else { + let mut failures = builder.delayed_failures.borrow_mut(); + failures.push(format!("{cmd:?}")); + false + } + } else { + true + } +} + +fn run_tests(builder: &Builder<'_>, cmd: &mut Command) -> bool { + cmd.stdout(Stdio::piped()); + + builder.verbose(&format!("running: {cmd:?}")); + + let mut process = cmd.spawn().unwrap(); + + // This runs until the stdout of the child is closed, which means the child exited. We don't + // run this on another thread since the builder is not Sync. + Renderer::new(process.stdout.take().unwrap(), builder).render_all(); + + let result = process.wait_with_output().unwrap(); + if !result.status.success() && builder.is_verbose() { + println!( + "\n\ncommand did not execute successfully: {cmd:?}\n\ + expected success, got: {}", + result.status + ); + } + + result.status.success() +} + +struct Renderer<'a> { + stdout: BufReader<ChildStdout>, + failures: Vec<TestOutcome>, + benches: Vec<BenchOutcome>, + builder: &'a Builder<'a>, + tests_count: Option<usize>, + executed_tests: usize, + terse_tests_in_line: usize, +} + +impl<'a> Renderer<'a> { + fn new(stdout: ChildStdout, builder: &'a Builder<'a>) -> Self { + Self { + stdout: BufReader::new(stdout), + benches: Vec::new(), + failures: Vec::new(), + builder, + tests_count: None, + executed_tests: 0, + terse_tests_in_line: 0, + } + } + + fn render_all(mut self) { + let mut line = String::new(); + loop { + line.clear(); + match self.stdout.read_line(&mut line) { + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(err) => panic!("failed to read output of test runner: {err}"), + } + if line.is_empty() { + break; + } + + match serde_json::from_str(&line) { + Ok(parsed) => self.render_message(parsed), + Err(_err) => { + // Handle non-JSON output, for example when --nocapture is passed. + print!("{line}"); + let _ = std::io::stdout().flush(); + } + } + } + } + + fn render_test_outcome(&mut self, outcome: Outcome<'_>, test: &TestOutcome) { + self.executed_tests += 1; + + #[cfg(feature = "build-metrics")] + self.builder.metrics.record_test( + &test.name, + match outcome { + Outcome::Ok | Outcome::BenchOk => crate::metrics::TestOutcome::Passed, + Outcome::Failed => crate::metrics::TestOutcome::Failed, + Outcome::Ignored { reason } => crate::metrics::TestOutcome::Ignored { + ignore_reason: reason.map(|s| s.to_string()), + }, + }, + ); + + if self.builder.config.verbose_tests { + self.render_test_outcome_verbose(outcome, test); + } else { + self.render_test_outcome_terse(outcome, test); + } + } + + fn render_test_outcome_verbose(&self, outcome: Outcome<'_>, test: &TestOutcome) { + print!("test {} ... ", test.name); + self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); + if let Some(exec_time) = test.exec_time { + print!(" ({exec_time:.2?})"); + } + println!(); + } + + fn render_test_outcome_terse(&mut self, outcome: Outcome<'_>, _: &TestOutcome) { + if self.terse_tests_in_line != 0 && self.terse_tests_in_line % TERSE_TESTS_PER_LINE == 0 { + if let Some(total) = self.tests_count { + let total = total.to_string(); + let executed = format!("{:>width$}", self.executed_tests - 1, width = total.len()); + print!(" {executed}/{total}"); + } + println!(); + self.terse_tests_in_line = 0; + } + + self.terse_tests_in_line += 1; + self.builder.colored_stdout(|stdout| outcome.write_short(stdout)).unwrap(); + let _ = std::io::stdout().flush(); + } + + fn render_suite_outcome(&self, outcome: Outcome<'_>, suite: &SuiteOutcome) { + // The terse output doesn't end with a newline, so we need to add it ourselves. + if !self.builder.config.verbose_tests { + println!(); + } + + if !self.failures.is_empty() { + println!("\nfailures:\n"); + for failure in &self.failures { + if let Some(stdout) = &failure.stdout { + println!("---- {} stdout ----", failure.name); + println!("{stdout}"); + } + } + + println!("\nfailures:"); + for failure in &self.failures { + println!(" {}", failure.name); + } + } + + if !self.benches.is_empty() { + println!("\nbenchmarks:"); + + let mut rows = Vec::new(); + for bench in &self.benches { + rows.push(( + &bench.name, + format!("{:.2?}/iter", Duration::from_nanos(bench.median)), + format!("+/- {:.2?}", Duration::from_nanos(bench.deviation)), + )); + } + + let max_0 = rows.iter().map(|r| r.0.len()).max().unwrap_or(0); + let max_1 = rows.iter().map(|r| r.1.len()).max().unwrap_or(0); + let max_2 = rows.iter().map(|r| r.2.len()).max().unwrap_or(0); + for row in &rows { + println!(" {:<max_0$} {:>max_1$} {:>max_2$}", row.0, row.1, row.2); + } + } + + print!("\ntest result: "); + self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); + println!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out; \ + finished in {:.2?}\n", + suite.passed, + suite.failed, + suite.ignored, + suite.measured, + suite.filtered_out, + Duration::from_secs_f64(suite.exec_time) + ); + } + + fn render_message(&mut self, message: Message) { + match message { + Message::Suite(SuiteMessage::Started { test_count }) => { + println!("\nrunning {test_count} tests"); + self.executed_tests = 0; + self.terse_tests_in_line = 0; + self.tests_count = Some(test_count); + } + Message::Suite(SuiteMessage::Ok(outcome)) => { + self.render_suite_outcome(Outcome::Ok, &outcome); + } + Message::Suite(SuiteMessage::Failed(outcome)) => { + self.render_suite_outcome(Outcome::Failed, &outcome); + } + Message::Bench(outcome) => { + // The formatting for benchmarks doesn't replicate 1:1 the formatting libtest + // outputs, mostly because libtest's formatting is broken in terse mode, which is + // the default used by our monorepo. We use a different formatting instead: + // successful benchmarks are just showed as "benchmarked"/"b", and the details are + // outputted at the bottom like failures. + let fake_test_outcome = TestOutcome { + name: outcome.name.clone(), + exec_time: None, + stdout: None, + message: None, + }; + self.render_test_outcome(Outcome::BenchOk, &fake_test_outcome); + self.benches.push(outcome); + } + Message::Test(TestMessage::Ok(outcome)) => { + self.render_test_outcome(Outcome::Ok, &outcome); + } + Message::Test(TestMessage::Ignored(outcome)) => { + self.render_test_outcome( + Outcome::Ignored { reason: outcome.message.as_deref() }, + &outcome, + ); + } + Message::Test(TestMessage::Failed(outcome)) => { + self.render_test_outcome(Outcome::Failed, &outcome); + self.failures.push(outcome); + } + Message::Test(TestMessage::Timeout { name }) => { + println!("test {name} has been running for a long time"); + } + Message::Test(TestMessage::Started) => {} // Not useful + } + } +} + +enum Outcome<'a> { + Ok, + BenchOk, + Failed, + Ignored { reason: Option<&'a str> }, +} + +impl Outcome<'_> { + fn write_short(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { + match self { + Outcome::Ok => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; + write!(writer, ".")?; + } + Outcome::BenchOk => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; + write!(writer, "b")?; + } + Outcome::Failed => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; + write!(writer, "F")?; + } + Outcome::Ignored { .. } => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; + write!(writer, "i")?; + } + } + writer.reset() + } + + fn write_long(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { + match self { + Outcome::Ok => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; + write!(writer, "ok")?; + } + Outcome::BenchOk => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; + write!(writer, "benchmarked")?; + } + Outcome::Failed => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; + write!(writer, "FAILED")?; + } + Outcome::Ignored { reason } => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; + write!(writer, "ignored")?; + if let Some(reason) = reason { + write!(writer, ", {reason}")?; + } + } + } + writer.reset() + } +} + +#[derive(serde_derive::Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Message { + Suite(SuiteMessage), + Test(TestMessage), + Bench(BenchOutcome), +} + +#[derive(serde_derive::Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum SuiteMessage { + Ok(SuiteOutcome), + Failed(SuiteOutcome), + Started { test_count: usize }, +} + +#[derive(serde_derive::Deserialize)] +struct SuiteOutcome { + passed: usize, + failed: usize, + ignored: usize, + measured: usize, + filtered_out: usize, + exec_time: f64, +} + +#[derive(serde_derive::Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum TestMessage { + Ok(TestOutcome), + Failed(TestOutcome), + Ignored(TestOutcome), + Timeout { name: String }, + Started, +} + +#[derive(serde_derive::Deserialize)] +struct BenchOutcome { + name: String, + median: u64, + deviation: u64, +} + +#[derive(serde_derive::Deserialize)] +struct TestOutcome { + name: String, + exec_time: Option<f64>, + stdout: Option<String>, + message: Option<String>, +} diff --git a/src/bootstrap/setup.rs b/src/bootstrap/setup.rs index 4480bce99..09f26862b 100644 --- a/src/bootstrap/setup.rs +++ b/src/bootstrap/setup.rs @@ -24,7 +24,7 @@ pub enum Profile { None, } -/// A list of historical hashes of `src/etc/vscode_settings.json`. +/// A list of historical hashes of `src/etc/rust_analyzer_settings.json`. /// New entries should be appended whenever this is updated so we can detect /// outdated vs. user-modified settings files. static SETTINGS_HASHES: &[&str] = &[ @@ -32,7 +32,7 @@ static SETTINGS_HASHES: &[&str] = &[ "56e7bf011c71c5d81e0bf42e84938111847a810eee69d906bba494ea90b51922", "af1b5efe196aed007577899db9dae15d6dbc923d6fa42fa0934e68617ba9bbe0", ]; -static VSCODE_SETTINGS: &str = include_str!("../etc/vscode_settings.json"); +static RUST_ANALYZER_SETTINGS: &str = include_str!("../etc/rust_analyzer_settings.json"); impl Profile { fn include_path(&self, src_path: &Path) -> PathBuf { @@ -489,7 +489,7 @@ undesirable, simply delete the `pre-push` file from .git/hooks." Ok(()) } -/// Sets up or displays `src/etc/vscode_settings.json` +/// Sets up or displays `src/etc/rust_analyzer_settings.json` #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] pub struct Vscode; @@ -580,10 +580,10 @@ fn create_vscode_settings_maybe(config: &Config) -> io::Result<()> { } _ => "Created", }; - fs::write(&vscode_settings, &VSCODE_SETTINGS)?; + fs::write(&vscode_settings, &RUST_ANALYZER_SETTINGS)?; println!("{verb} `.vscode/settings.json`"); } else { - println!("\n{VSCODE_SETTINGS}"); + println!("\n{RUST_ANALYZER_SETTINGS}"); } Ok(()) } diff --git a/src/bootstrap/setup/tests.rs b/src/bootstrap/setup/tests.rs index dcf9d18e6..0fe6e4a46 100644 --- a/src/bootstrap/setup/tests.rs +++ b/src/bootstrap/setup/tests.rs @@ -1,14 +1,14 @@ -use super::{SETTINGS_HASHES, VSCODE_SETTINGS}; +use super::{RUST_ANALYZER_SETTINGS, SETTINGS_HASHES}; use sha2::Digest; #[test] fn check_matching_settings_hash() { let mut hasher = sha2::Sha256::new(); - hasher.update(&VSCODE_SETTINGS); + hasher.update(&RUST_ANALYZER_SETTINGS); let hash = hex::encode(hasher.finalize().as_slice()); assert_eq!( &hash, SETTINGS_HASHES.last().unwrap(), - "Update `SETTINGS_HASHES` with the new hash of `src/etc/vscode_settings.json`" + "Update `SETTINGS_HASHES` with the new hash of `src/etc/rust_analyzer_settings.json`" ); } diff --git a/src/bootstrap/suggest.rs b/src/bootstrap/suggest.rs new file mode 100644 index 000000000..ff20ebec2 --- /dev/null +++ b/src/bootstrap/suggest.rs @@ -0,0 +1,80 @@ +#![cfg_attr(feature = "build-metrics", allow(unused))] + +use std::str::FromStr; + +use std::path::PathBuf; + +use crate::{ + builder::{Builder, Kind}, + tool::Tool, +}; + +#[cfg(feature = "build-metrics")] +pub fn suggest(builder: &Builder<'_>, run: bool) { + panic!("`x suggest` is not supported with `build-metrics`") +} + +/// Suggests a list of possible `x.py` commands to run based on modified files in branch. +#[cfg(not(feature = "build-metrics"))] +pub fn suggest(builder: &Builder<'_>, run: bool) { + let suggestions = + builder.tool_cmd(Tool::SuggestTests).output().expect("failed to run `suggest-tests` tool"); + + if !suggestions.status.success() { + println!("failed to run `suggest-tests` tool ({})", suggestions.status); + println!( + "`suggest_tests` stdout:\n{}`suggest_tests` stderr:\n{}", + String::from_utf8(suggestions.stdout).unwrap(), + String::from_utf8(suggestions.stderr).unwrap() + ); + panic!("failed to run `suggest-tests`"); + } + + let suggestions = String::from_utf8(suggestions.stdout).unwrap(); + let suggestions = suggestions + .lines() + .map(|line| { + let mut sections = line.split_ascii_whitespace(); + + // this code expects one suggestion per line in the following format: + // <x_subcommand> {some number of flags} [optional stage number] + let cmd = sections.next().unwrap(); + let stage = sections.next_back().map(|s| str::parse(s).ok()).flatten(); + let paths: Vec<PathBuf> = sections.map(|p| PathBuf::from_str(p).unwrap()).collect(); + + (cmd, stage, paths) + }) + .collect::<Vec<_>>(); + + if !suggestions.is_empty() { + println!("==== SUGGESTIONS ===="); + for sug in &suggestions { + print!("x {} ", sug.0); + if let Some(stage) = sug.1 { + print!("--stage {stage} "); + } + + for path in &sug.2 { + print!("{} ", path.display()); + } + println!(); + } + println!("====================="); + } else { + println!("No suggestions found!"); + return; + } + + if run { + for sug in suggestions { + let mut build = builder.build.clone(); + + let builder = + Builder::new_standalone(&mut build, Kind::parse(&sug.0).unwrap(), sug.2, sug.1); + + builder.execute_cli() + } + } else { + println!("help: consider using the `--run` flag to automatically run suggested tests"); + } +} diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index fc850a22b..7fa8a4d9d 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -318,6 +318,7 @@ impl<'a> Tarball<'a> { assert!(!formats.is_empty(), "dist.compression-formats can't be empty"); cmd.arg("--compression-formats").arg(formats.join(",")); } + cmd.args(&["--compression-profile", &self.builder.config.dist_compression_profile]); self.builder.run(&mut cmd); // Ensure there are no symbolic links in the tarball. In particular, diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index b4f1506dc..aedf1ecab 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -19,10 +19,11 @@ use crate::config::TargetSelection; use crate::dist; use crate::doc::DocumentationFormat; use crate::flags::Subcommand; -use crate::native; +use crate::llvm; +use crate::render_tests::add_flags_and_try_run_tests; use crate::tool::{self, SourceType, Tool}; use crate::toolstate::ToolState; -use crate::util::{self, add_link_lib_path, dylib_path, dylib_path_var, output, t}; +use crate::util::{self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date}; use crate::{envify, CLang, DocTests, GitRepo, Mode}; const ADB_TEST_DIR: &str = "/data/local/tmp/work"; @@ -123,7 +124,43 @@ impl Step for CrateJsonDocLint { SourceType::InTree, &[], ); - try_run(builder, &mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct SuggestTestsCrate { + host: TargetSelection, +} + +impl Step for SuggestTestsCrate { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/suggest-tests") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(SuggestTestsCrate { host: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + + let suggest_tests = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + "src/tools/suggest-tests", + SourceType::InTree, + &[], + ); + add_flags_and_try_run_tests(builder, &mut suggest_tests.into()); } } @@ -172,7 +209,7 @@ You can skip linkcheck with --exclude src/tools/linkchecker" SourceType::InTree, &[], ); - try_run(builder, &mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); // Build all the default documentation. builder.default_doc(&[]); @@ -333,7 +370,7 @@ impl Step for Cargo { cargo.env("PATH", &path_for_cargo(builder, compiler)); - try_run(builder, &mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); } } @@ -392,7 +429,7 @@ impl Step for RustAnalyzer { cargo.add_rustc_lib_path(builder, compiler); cargo.arg("--").args(builder.config.cmd.test_args()); - builder.run(&mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); } } @@ -445,7 +482,7 @@ impl Step for Rustfmt { cargo.add_rustc_lib_path(builder, compiler); - builder.run(&mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); } } @@ -496,7 +533,7 @@ impl Step for RustDemangler { cargo.add_rustc_lib_path(builder, compiler); - builder.run(&mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); } } @@ -637,6 +674,8 @@ impl Step for Miri { // Forward test filters. cargo.arg("--").args(builder.config.cmd.test_args()); + // This can NOT be `add_flags_and_try_run_tests` since the Miri test runner + // does not understand those flags! let mut cargo = Command::from(cargo); builder.run(&mut cargo); @@ -694,7 +733,7 @@ impl Step for CompiletestTest { /// Runs `cargo test` for compiletest. fn run(self, builder: &Builder<'_>) { let host = self.host; - let compiler = builder.compiler(0, host); + let compiler = builder.compiler(1, host); // We need `ToolStd` for the locally-built sysroot because // compiletest uses unstable features of the `test` crate. @@ -711,7 +750,7 @@ impl Step for CompiletestTest { ); cargo.allow_features("test"); - try_run(builder, &mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); } } @@ -1064,6 +1103,8 @@ impl Step for RustdocGUI { cargo.env("RUSTDOCFLAGS", "-Zunstable-options --generate-link-to-definition"); } else if entry.file_name() == "scrape_examples" { cargo.arg("-Zrustdoc-scrape-examples"); + } else if entry.file_name() == "extend_css" { + cargo.env("RUSTDOCFLAGS", &format!("--extend-css extra.css")); } builder.run(&mut cargo); } @@ -1118,7 +1159,11 @@ impl Step for Tidy { cmd.arg(&builder.src); cmd.arg(&builder.initial_cargo); cmd.arg(&builder.out); - cmd.arg(builder.jobs().to_string()); + // Tidy is heavily IO constrained. Still respect `-j`, but use a higher limit if `jobs` hasn't been configured. + let jobs = builder.config.jobs.unwrap_or_else(|| { + 8 * std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 + }); + cmd.arg(jobs.to_string()); if builder.is_verbose() { cmd.arg("--verbose"); } @@ -1129,7 +1174,7 @@ impl Step for Tidy { if builder.config.channel == "dev" || builder.config.channel == "nightly" { builder.info("fmt check"); if builder.initial_rustfmt().is_none() { - let inferred_rustfmt_dir = builder.config.initial_rustc.parent().unwrap(); + let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); eprintln!( "\ error: no `rustfmt` binary found in {PATH} @@ -1189,7 +1234,7 @@ impl Step for TidySelfTest { SourceType::InTree, &[], ); - try_run(builder, &mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); } } @@ -1430,11 +1475,11 @@ note: if you're sure you want to do this, please open an issue as to why. In the builder.ensure(compile::Std::new(compiler, compiler.host)); // Also provide `rust_test_helpers` for the host. - builder.ensure(native::TestHelpers { target: compiler.host }); + builder.ensure(TestHelpers { target: compiler.host }); // As well as the target, except for plain wasm32, which can't build it if !target.contains("wasm") || target.contains("emscripten") { - builder.ensure(native::TestHelpers { target }); + builder.ensure(TestHelpers { target }); } builder.ensure(RemoteCopyLibs { compiler, target }); @@ -1531,7 +1576,10 @@ note: if you're sure you want to do this, please open an issue as to why. In the flags.extend(builder.config.cmd.rustc_args().iter().map(|s| s.to_string())); if let Some(linker) = builder.linker(target) { - cmd.arg("--linker").arg(linker); + cmd.arg("--target-linker").arg(linker); + } + if let Some(linker) = builder.linker(compiler.host) { + cmd.arg("--host-linker").arg(linker); } let mut hostflags = flags.clone(); @@ -1616,15 +1664,13 @@ note: if you're sure you want to do this, please open an issue as to why. In the cmd.arg("--verbose"); } - if !builder.config.verbose_tests { - cmd.arg("--quiet"); - } + cmd.arg("--json"); let mut llvm_components_passed = false; let mut copts_passed = false; if builder.config.llvm_enabled() { - let native::LlvmResult { llvm_config, .. } = - builder.ensure(native::Llvm { target: builder.config.build }); + let llvm::LlvmResult { llvm_config, .. } = + builder.ensure(llvm::Llvm { target: builder.config.build }); if !builder.config.dry_run() { let llvm_version = output(Command::new(&llvm_config).arg("--version")); let llvm_components = output(Command::new(&llvm_config).arg("--components")); @@ -1662,7 +1708,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the // If LLD is available, add it to the PATH if builder.config.lld_enabled { let lld_install_root = - builder.ensure(native::Lld { target: builder.config.build }); + builder.ensure(llvm::Lld { target: builder.config.build }); let lld_bin_path = lld_install_root.join("bin"); @@ -1769,7 +1815,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the suite, mode, &compiler.host, target )); let _time = util::timeit(&builder); - try_run(builder, &mut cmd); + crate::render_tests::try_run_tests(builder, &mut cmd); if let Some(compare_mode) = compare_mode { cmd.arg("--compare-mode").arg(compare_mode); @@ -1778,7 +1824,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the suite, mode, compare_mode, &compiler.host, target )); let _time = util::timeit(&builder); - try_run(builder, &mut cmd); + crate::render_tests::try_run_tests(builder, &mut cmd); } } } @@ -2141,7 +2187,7 @@ impl Step for Crate { compile::std_cargo(builder, target, compiler.stage, &mut cargo); } Mode::Rustc => { - compile::rustc_cargo(builder, &mut cargo, target); + compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); } _ => panic!("can only test libraries"), }; @@ -2180,9 +2226,8 @@ impl Step for Crate { cargo.arg("--"); cargo.args(&builder.config.cmd.test_args()); - if !builder.config.verbose_tests { - cargo.arg("--quiet"); - } + cargo.arg("-Z").arg("unstable-options"); + cargo.arg("--format").arg("json"); if target.contains("emscripten") { cargo.env( @@ -2210,7 +2255,7 @@ impl Step for Crate { target )); let _time = util::timeit(&builder); - try_run(builder, &mut cargo.into()); + crate::render_tests::try_run_tests(builder, &mut cargo.into()); } } @@ -2330,7 +2375,7 @@ impl Step for CrateRustdoc { )); let _time = util::timeit(&builder); - try_run(builder, &mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); } } @@ -2391,17 +2436,13 @@ impl Step for CrateRustdocJsonTypes { cargo.arg("'-Ctarget-feature=-crt-static'"); } - if !builder.config.verbose_tests { - cargo.arg("--quiet"); - } - builder.info(&format!( "{} rustdoc-json-types stage{} ({} -> {})", test_kind, compiler.stage, &compiler.host, target )); let _time = util::timeit(&builder); - try_run(builder, &mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); } } @@ -2570,7 +2611,7 @@ impl Step for Bootstrap { // rustbuild tests are racy on directory creation so just run them one at a time. // Since there's not many this shouldn't be a problem. cmd.arg("--test-threads=1"); - try_run(builder, &mut cmd); + add_flags_and_try_run_tests(builder, &mut cmd); } fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -2651,7 +2692,7 @@ impl Step for ReplacePlaceholderTest { SourceType::InTree, &[], ); - try_run(builder, &mut cargo.into()); + add_flags_and_try_run_tests(builder, &mut cargo.into()); } fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -2695,3 +2736,123 @@ impl Step for LintDocs { }); } } + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustInstaller; + +impl Step for RustInstaller { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + /// Ensure the version placeholder replacement tool builds + fn run(self, builder: &Builder<'_>) { + builder.info("test rust-installer"); + + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + "src/tools/rust-installer", + SourceType::InTree, + &[], + ); + try_run(builder, &mut cargo.into()); + + // We currently don't support running the test.sh script outside linux(?) environments. + // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a + // set of scripts, which will likely allow dropping this if. + if bootstrap_host != "x86_64-unknown-linux-gnu" { + return; + } + + let mut cmd = + std::process::Command::new(builder.src.join("src/tools/rust-installer/test.sh")); + let tmpdir = testdir(builder, compiler.host).join("rust-installer"); + let _ = std::fs::remove_dir_all(&tmpdir); + let _ = std::fs::create_dir_all(&tmpdir); + cmd.current_dir(&tmpdir); + cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target")); + cmd.env("CARGO", &builder.initial_cargo); + cmd.env("RUSTC", &builder.initial_rustc); + cmd.env("TMP_DIR", &tmpdir); + try_run(builder, &mut cmd); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-installer") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Self); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct TestHelpers { + pub target: TargetSelection, +} + +impl Step for TestHelpers { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("tests/auxiliary/rust_test_helpers.c") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(TestHelpers { target: run.target }) + } + + /// Compiles the `rust_test_helpers.c` library which we used in various + /// `run-pass` tests for ABI testing. + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run() { + return; + } + // The x86_64-fortanix-unknown-sgx target doesn't have a working C + // toolchain. However, some x86_64 ELF objects can be linked + // without issues. Use this hack to compile the test helpers. + let target = if self.target == "x86_64-fortanix-unknown-sgx" { + TargetSelection::from_user("x86_64-unknown-linux-gnu") + } else { + self.target + }; + let dst = builder.test_helpers_out(target); + let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); + if up_to_date(&src, &dst.join("librust_test_helpers.a")) { + return; + } + + builder.info("Building test helpers"); + t!(fs::create_dir_all(&dst)); + let mut cfg = cc::Build::new(); + // FIXME: Workaround for https://github.com/emscripten-core/emscripten/issues/9013 + if target.contains("emscripten") { + cfg.pic(false); + } + + // We may have found various cross-compilers a little differently due to our + // extra configuration, so inform cc of these compilers. Note, though, that + // on MSVC we still need cc's detection of env vars (ugh). + if !target.contains("msvc") { + if let Some(ar) = builder.ar(target) { + cfg.archiver(ar); + } + cfg.compiler(builder.cc(target)); + } + cfg.cargo_metadata(false) + .out_dir(&dst) + .target(&target.triple) + .host(&builder.config.build.triple) + .opt_level(0) + .warnings(false) + .debug(false) + .file(builder.src.join("tests/auxiliary/rust_test_helpers.c")) + .compile("rust_test_helpers"); + } +} diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 3c9a154da..d1fd2e8c4 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -320,7 +320,7 @@ pub fn prepare_tool_cargo( cargo.env("CFG_RELEASE_NUM", &builder.version); cargo.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); - let info = GitInfo::new(builder.config.ignore_git, &dir); + let info = GitInfo::new(builder.config.omit_git_hash, &dir); if let Some(sha) = info.sha() { cargo.env("CFG_COMMIT_HASH", sha); } @@ -433,6 +433,7 @@ bootstrap_tool!( ReplaceVersionPlaceholder, "src/tools/replace-version-placeholder", "replace-version-placeholder"; CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata"; GenerateCopyright, "src/tools/generate-copyright", "generate-copyright"; + SuggestTests, "src/tools/suggest-tests", "suggest-tests"; ); #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index 93e53d383..2e1adbf63 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -146,100 +146,9 @@ pub fn symlink_dir(config: &Config, src: &Path, dest: &Path) -> io::Result<()> { fs::symlink(src, dest) } - // Creating a directory junction on windows involves dealing with reparse - // points and the DeviceIoControl function, and this code is a skeleton of - // what can be found here: - // - // http://www.flexhex.com/docs/articles/hard-links.phtml #[cfg(windows)] fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> { - use std::ffi::OsStr; - use std::os::windows::ffi::OsStrExt; - use std::ptr; - - use winapi::shared::minwindef::{DWORD, WORD}; - use winapi::um::fileapi::{CreateFileW, OPEN_EXISTING}; - use winapi::um::handleapi::CloseHandle; - use winapi::um::ioapiset::DeviceIoControl; - use winapi::um::winbase::{FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT}; - use winapi::um::winioctl::FSCTL_SET_REPARSE_POINT; - use winapi::um::winnt::{ - FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, GENERIC_WRITE, - IO_REPARSE_TAG_MOUNT_POINT, MAXIMUM_REPARSE_DATA_BUFFER_SIZE, WCHAR, - }; - - #[allow(non_snake_case)] - #[repr(C)] - struct REPARSE_MOUNTPOINT_DATA_BUFFER { - ReparseTag: DWORD, - ReparseDataLength: DWORD, - Reserved: WORD, - ReparseTargetLength: WORD, - ReparseTargetMaximumLength: WORD, - Reserved1: WORD, - ReparseTarget: WCHAR, - } - - fn to_u16s<S: AsRef<OsStr>>(s: S) -> io::Result<Vec<u16>> { - Ok(s.as_ref().encode_wide().chain(Some(0)).collect()) - } - - // We're using low-level APIs to create the junction, and these are more - // picky about paths. For example, forward slashes cannot be used as a - // path separator, so we should try to canonicalize the path first. - let target = fs::canonicalize(target)?; - - fs::create_dir(junction)?; - - let path = to_u16s(junction)?; - - unsafe { - let h = CreateFileW( - path.as_ptr(), - GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - ptr::null_mut(), - OPEN_EXISTING, - FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, - ptr::null_mut(), - ); - - #[repr(C, align(8))] - struct Align8<T>(T); - let mut data = Align8([0u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE as usize]); - let db = data.0.as_mut_ptr() as *mut REPARSE_MOUNTPOINT_DATA_BUFFER; - let buf = core::ptr::addr_of_mut!((*db).ReparseTarget) as *mut u16; - let mut i = 0; - // FIXME: this conversion is very hacky - let v = br"\??\"; - let v = v.iter().map(|x| *x as u16); - for c in v.chain(target.as_os_str().encode_wide().skip(4)) { - *buf.offset(i) = c; - i += 1; - } - *buf.offset(i) = 0; - i += 1; - (*db).ReparseTag = IO_REPARSE_TAG_MOUNT_POINT; - (*db).ReparseTargetMaximumLength = (i * 2) as WORD; - (*db).ReparseTargetLength = ((i - 1) * 2) as WORD; - (*db).ReparseDataLength = (*db).ReparseTargetLength as DWORD + 12; - - let mut ret = 0; - let res = DeviceIoControl( - h as *mut _, - FSCTL_SET_REPARSE_POINT, - db.cast(), - (*db).ReparseDataLength + 8, - ptr::null_mut(), - 0, - &mut ret, - ptr::null_mut(), - ); - - let out = if res == 0 { Err(io::Error::last_os_error()) } else { Ok(()) }; - CloseHandle(h); - out - } + junction::create(&target, &junction) } } |