diff options
Diffstat (limited to 'src/bootstrap')
31 files changed, 1028 insertions, 501 deletions
diff --git a/src/bootstrap/CHANGELOG.md b/src/bootstrap/CHANGELOG.md index 64b74ecc9..4105fa5ec 100644 --- a/src/bootstrap/CHANGELOG.md +++ b/src/bootstrap/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Several unsupported `./configure` options have been removed: `optimize`, `parallel-compiler`. These can still be enabled with `--set`, although it isn't recommended. - `remote-test-server`'s `verbose` argument has been removed in favor of the `--verbose` flag - `remote-test-server`'s `remote` argument has been removed in favor of the `--bind` flag. Use `--bind 0.0.0.0:12345` to replicate the behavior of the `remote` argument. +- `x.py fmt` now formats only files modified between the merge-base of HEAD and the last commit in the master branch of the rust-lang repository and the current working directory. To restore old behaviour, use `x.py fmt .`. The check mode is not affected by this change. [#105702](https://github.com/rust-lang/rust/pull/105702) ### Non-breaking changes diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index efe8ae316..4a0ba5925 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -36,6 +36,7 @@ dependencies = [ name = "bootstrap" version = "0.0.0" dependencies = [ + "build_helper", "cc", "cmake", "fd-lock", @@ -71,6 +72,10 @@ dependencies = [ ] [[package]] +name = "build_helper" +version = "0.1.0" + +[[package]] name = "cc" version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index ccc7ec1fc..22ceeca94 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -29,12 +29,8 @@ name = "sccache-plus-cl" path = "bin/sccache-plus-cl.rs" test = false -[[bin]] -name = "llvm-config-wrapper" -path = "bin/llvm-config-wrapper.rs" -test = false - [dependencies] +build_helper = { path = "../tools/build_helper" } cmake = "0.1.38" fd-lock = "3.0.8" filetime = "0.2" diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 985727bdd..8dce9e79e 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -59,10 +59,10 @@ The script accepts commands, flags, and arguments to determine what to do: ./x.py test tidy # execute the UI test suite - ./x.py test src/test/ui + ./x.py test tests/ui # execute only some tests in the UI test suite - ./x.py test src/test/ui --test-args substring-of-test-name + ./x.py test tests/ui --test-args substring-of-test-name # execute tests in the standard library in stage0 ./x.py test --stage 0 library/std @@ -80,18 +80,12 @@ The script accepts commands, flags, and arguments to determine what to do: ## Configuring rustbuild -There are currently two methods for configuring the rustbuild build system. - -First, rustbuild offers a TOML-based configuration system with a `config.toml` +rustbuild offers a TOML-based configuration system with a `config.toml` file. An example of this configuration can be found at `config.toml.example`, and the configuration file can also be passed as `--config path/to/config.toml` if the build system is being invoked manually (via the python script). -Next, the `./configure` options serialized in `config.mk` will be -parsed and read. That is, if any `./configure` options are passed, they'll be -handled naturally. `./configure` should almost never be used for local -installations, and is primarily useful for CI. Prefer to customize behavior -using `config.toml`. +You can generate a config.toml using `./configure` options if you want to automate creating the file without having to edit it. Finally, rustbuild makes use of the [cc-rs crate] which has [its own method][env-vars] of configuring C compilers and C flags via environment diff --git a/src/bootstrap/bin/llvm-config-wrapper.rs b/src/bootstrap/bin/llvm-config-wrapper.rs deleted file mode 100644 index 89984bb55..000000000 --- a/src/bootstrap/bin/llvm-config-wrapper.rs +++ /dev/null @@ -1,24 +0,0 @@ -// The sheer existence of this file is an awful hack. See the comments in -// `src/bootstrap/native.rs` for why this is needed when compiling LLD. - -use std::env; -use std::io::{self, Write}; -use std::process::{self, Command, Stdio}; - -fn main() { - let real_llvm_config = env::var_os("LLVM_CONFIG_REAL").unwrap(); - let mut cmd = Command::new(real_llvm_config); - cmd.args(env::args().skip(1)).stderr(Stdio::piped()); - let output = cmd.output().expect("failed to spawn llvm-config"); - let mut stdout = String::from_utf8_lossy(&output.stdout); - - if let Ok(to_replace) = env::var("LLVM_CONFIG_SHIM_REPLACE") { - if let Ok(replace_with) = env::var("LLVM_CONFIG_SHIM_REPLACE_WITH") { - stdout = stdout.replace(&to_replace, &replace_with).into(); - } - } - - print!("{}", stdout.replace("\\", "/")); - io::stdout().flush().unwrap(); - process::exit(output.status.code().unwrap_or(1)); -} diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index 1d5260228..9611c866d 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -154,6 +154,41 @@ fn main() { cmd.arg("-Z").arg("force-unstable-if-unmarked"); } + // allow-features is handled from within this rustc wrapper because of + // issues with build scripts. Some packages use build scripts to + // dynamically detect if certain nightly features are available. + // There are different ways this causes problems: + // + // * rustix runs `rustc` on a small test program to see if the feature is + // available (and sets a `cfg` if it is). It does not honor + // CARGO_ENCODED_RUSTFLAGS. + // * proc-macro2 detects if `rustc -vV` says "nighty" or "dev" and enables + // nightly features. It will scan CARGO_ENCODED_RUSTFLAGS for + // -Zallow-features. Unfortunately CARGO_ENCODED_RUSTFLAGS is not set + // for build-dependencies when --target is used. + // + // The issues above means we can't just use RUSTFLAGS, and we can't use + // `cargo -Zallow-features=…`. Passing it through here ensures that it + // always gets set. Unfortunately that also means we need to enable more + // features than we really want (like those for proc-macro2), but there + // isn't much of a way around it. + // + // I think it is unfortunate that build scripts are doing this at all, + // since changes to nightly features can cause crates to break even if the + // user didn't want or care about the use of the nightly features. I think + // nightly features should be opt-in only. Unfortunately the dynamic + // checks are now too wide spread that we just need to deal with it. + // + // If you want to try to remove this, I suggest working with the crate + // authors to remove the dynamic checking. Another option is to pursue + // https://github.com/rust-lang/cargo/issues/11244 and + // https://github.com/rust-lang/cargo/issues/4423, which will likely be + // very difficult, but could help expose -Zallow-features into build + // scripts so they could try to honor them. + if let Ok(allow_features) = env::var("RUSTC_ALLOW_FEATURES") { + cmd.arg(format!("-Zallow-features={allow_features}")); + } + if let Ok(flags) = env::var("MAGIC_EXTRA_RUSTFLAGS") { for flag in flags.split(' ') { cmd.arg(flag); diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 2d5018d93..9cf43fc7a 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -753,6 +753,9 @@ class RustBuild(object): target_features += ["-crt-static"] if target_features: env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features)) + target_linker = self.get_toml("linker", build_section) + if target_linker is not None: + env["RUSTFLAGS"] += " -C linker=" + target_linker env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros" if self.get_toml("deny-warnings", "rust") != "false": diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 8ee6d49da..b4fc1d4f2 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -13,17 +13,18 @@ use std::time::{Duration, Instant}; use crate::cache::{Cache, Interned, INTERNER}; use crate::config::{SplitDebuginfo, TargetSelection}; -use crate::dist; use crate::doc; use crate::flags::{Color, Subcommand}; use crate::install; use crate::native; use crate::run; +use crate::setup; use crate::test; use crate::tool::{self, SourceType}; use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; use crate::EXTRA_CHECK_CFGS; use crate::{check, compile, Crate}; +use crate::{clean, dist}; use crate::{Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; @@ -95,6 +96,35 @@ impl RunConfig<'_> { pub fn build_triple(&self) -> TargetSelection { self.builder.build.build } + + /// Return a list of crate names selected by `run.paths`. + pub fn cargo_crates_in_set(&self) -> Interned<Vec<String>> { + let mut crates = Vec::new(); + for krate in &self.paths { + let path = krate.assert_single_path(); + let crate_name = self.builder.crate_paths[&path.path]; + crates.push(crate_name.to_string()); + } + INTERNER.intern_list(crates) + } +} + +/// A description of the crates in this set, suitable for passing to `builder.info`. +/// +/// `crates` should be generated by [`RunConfig::cargo_crates_in_set`]. +pub fn crate_description(crates: &[impl AsRef<str>]) -> String { + if crates.is_empty() { + return "".into(); + } + + let mut descr = String::from(" {"); + descr.push_str(crates[0].as_ref()); + for krate in &crates[1..] { + descr.push_str(", "); + descr.push_str(krate.as_ref()); + } + descr.push('}'); + descr } struct StepDescription { @@ -161,9 +191,9 @@ pub enum PathSet { /// A "suite" of paths. /// /// These can match as a path suffix (like `Set`), or as a prefix. For - /// example, a command-line value of `src/test/ui/abi/variadic-ffi.rs` - /// will match `src/test/ui`. A command-line value of `ui` would also - /// match `src/test/ui`. + /// example, a command-line value of `tests/ui/abi/variadic-ffi.rs` + /// will match `tests/ui`. A command-line value of `ui` would also + /// match `tests/ui`. Suite(TaskPath), } @@ -433,8 +463,11 @@ impl<'a> ShouldRun<'a> { // single alias, which does not correspond to any on-disk path pub fn alias(mut self, alias: &str) -> Self { + // exceptional case for `Kind::Setup` because its `library` + // and `compiler` options would otherwise naively match with + // `compiler` and `library` folders respectively. assert!( - !self.builder.src.join(alias).exists(), + self.kind == Kind::Setup || !self.builder.src.join(alias).exists(), "use `builder.path()` for real paths: {}", alias ); @@ -629,6 +662,7 @@ impl<'a> Builder<'a> { crate::toolstate::ToolStateCheck, test::ExpandYamlAnchors, test::Tidy, + test::TidySelfTest, test::Ui, test::RunPassValgrind, test::MirOpt, @@ -698,6 +732,7 @@ impl<'a> Builder<'a> { doc::RustdocBook, doc::RustByExample, doc::RustcBook, + doc::Cargo, doc::CargoBook, doc::Clippy, doc::ClippyBook, @@ -744,6 +779,7 @@ impl<'a> Builder<'a> { install::RustDemangler, install::Clippy, install::Miri, + install::LlvmTools, install::Analysis, install::Src, install::Rustc @@ -757,8 +793,10 @@ impl<'a> Builder<'a> { run::CollectLicenseMetadata, run::GenerateCopyright, ), - // These commands either don't use paths, or they're special-cased in Build::build() - Kind::Clean | Kind::Format | Kind::Setup => vec![], + Kind::Setup => describe!(setup::Profile), + Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std), + // special-cased in Build::build() + Kind::Format => vec![], } } @@ -820,10 +858,12 @@ impl<'a> Builder<'a> { Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]), Subcommand::Install { ref paths } => (Kind::Install, &paths[..]), Subcommand::Run { ref paths, .. } => (Kind::Run, &paths[..]), + Subcommand::Clean { ref paths, .. } => (Kind::Clean, &paths[..]), Subcommand::Format { .. } => (Kind::Format, &[][..]), - Subcommand::Clean { .. } | Subcommand::Setup { .. } => { - panic!() - } + Subcommand::Setup { profile: ref path } => ( + Kind::Setup, + path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)), + ), }; Self::new_internal(build, kind, paths.to_owned()) @@ -1058,7 +1098,7 @@ impl<'a> Builder<'a> { /// check build or dry-run, where there's no need to build all of LLVM. fn llvm_config(&self, target: TargetSelection) -> Option<PathBuf> { if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() { - let llvm_config = self.ensure(native::Llvm { target }); + let native::LlvmResult { llvm_config, .. } = self.ensure(native::Llvm { target }); if llvm_config.is_file() { return Some(llvm_config); } @@ -1066,6 +1106,62 @@ impl<'a> Builder<'a> { None } + /// Like `cargo`, but only passes flags that are valid for all commands. + pub fn bare_cargo( + &self, + compiler: Compiler, + mode: Mode, + target: TargetSelection, + cmd: &str, + ) -> Command { + let mut cargo = Command::new(&self.initial_cargo); + // Run cargo from the source root so it can find .cargo/config. + // This matters when using vendoring and the working directory is outside the repository. + cargo.current_dir(&self.src); + + let out_dir = self.stage_out(compiler, mode); + cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd); + + // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger` + // from out of tree it shouldn't matter, since x.py is only used for + // building in-tree. + let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"]; + match self.build.config.color { + Color::Always => { + cargo.arg("--color=always"); + for log in &color_logs { + cargo.env(log, "always"); + } + } + Color::Never => { + cargo.arg("--color=never"); + for log in &color_logs { + cargo.env(log, "never"); + } + } + Color::Auto => {} // nothing to do + } + + if cmd != "install" { + cargo.arg("--target").arg(target.rustc_target_arg()); + } else { + assert_eq!(target, compiler.host); + } + + if self.config.rust_optimize { + // FIXME: cargo bench/install do not accept `--release` + if cmd != "bench" && cmd != "install" { + cargo.arg("--release"); + } + } + + // Remove make-related flags to ensure Cargo can correctly set things up + cargo.env_remove("MAKEFLAGS"); + cargo.env_remove("MFLAGS"); + + cargo + } + /// Prepares an invocation of `cargo` to be run. /// /// This will create a `Command` that represents a pending execution of @@ -1081,11 +1177,8 @@ impl<'a> Builder<'a> { target: TargetSelection, cmd: &str, ) -> Cargo { - let mut cargo = Command::new(&self.initial_cargo); + let mut cargo = self.bare_cargo(compiler, mode, target, cmd); let out_dir = self.stage_out(compiler, mode); - // Run cargo from the source root so it can find .cargo/config. - // This matters when using vendoring and the working directory is outside the repository. - cargo.current_dir(&self.src); // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, // so we need to explicitly clear out if they've been updated. @@ -1110,8 +1203,6 @@ impl<'a> Builder<'a> { self.clear_if_dirty(&my_out, &rustdoc); } - cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd); - let profile_var = |name: &str| { let profile = if self.config.rust_optimize { "RELEASE" } else { "DEV" }; format!("CARGO_PROFILE_{}_{}", profile, name) @@ -1124,32 +1215,6 @@ impl<'a> Builder<'a> { cargo.env("REAL_LIBRARY_PATH", e); } - // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger` - // from out of tree it shouldn't matter, since x.py is only used for - // building in-tree. - let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"]; - match self.build.config.color { - Color::Always => { - cargo.arg("--color=always"); - for log in &color_logs { - cargo.env(log, "always"); - } - } - Color::Never => { - cargo.arg("--color=never"); - for log in &color_logs { - cargo.env(log, "never"); - } - } - Color::Auto => {} // nothing to do - } - - if cmd != "install" { - cargo.arg("--target").arg(target.rustc_target_arg()); - } else { - assert_eq!(target, compiler.host); - } - // Set a flag for `check`/`clippy`/`fix`, so that certain build // scripts can do less work (i.e. not building/requiring LLVM). if cmd == "check" || cmd == "clippy" || cmd == "fix" { @@ -1316,23 +1381,31 @@ impl<'a> Builder<'a> { // this), as well as #63012 which is the tracking issue for this // feature on the rustc side. cargo.arg("-Zbinary-dep-depinfo"); - match mode { - Mode::ToolBootstrap => { - // Restrict the allowed features to those passed by rustbuild, so we don't depend on nightly accidentally. - rustflags.arg("-Zallow-features=binary-dep-depinfo"); + let allow_features = match mode { + Mode::ToolBootstrap | Mode::ToolStd => { + // Restrict the allowed features so we don't depend on nightly + // accidentally. + // + // binary-dep-depinfo is used by rustbuild itself for all + // compilations. + // + // Lots of tools depend on proc_macro2 and proc-macro-error. + // Those have build scripts which assume nightly features are + // available if the `rustc` version is "nighty" or "dev". See + // bin/rustc.rs for why that is a problem. Instead of labeling + // those features for each individual tool that needs them, + // just blanket allow them here. + // + // If this is ever removed, be sure to add something else in + // its place to keep the restrictions in place (or make a way + // to unset RUSTC_BOOTSTRAP). + "binary-dep-depinfo,proc_macro_span,proc_macro_span_shrink,proc_macro_diagnostic" + .to_string() } - Mode::ToolStd => { - // Right now this is just compiletest and a few other tools that build on stable. - // Allow them to use `feature(test)`, but nothing else. - rustflags.arg("-Zallow-features=binary-dep-depinfo,test,proc_macro_internals,proc_macro_diagnostic,proc_macro_span"); - } - Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => {} - } + Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => String::new(), + }; cargo.arg("-j").arg(self.jobs().to_string()); - // Remove make-related flags to ensure Cargo can correctly set things up - cargo.env_remove("MAKEFLAGS"); - cargo.env_remove("MFLAGS"); // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005 // Force cargo to output binaries with disambiguating hashes in the name @@ -1816,13 +1889,6 @@ impl<'a> Builder<'a> { } } - if self.config.rust_optimize { - // FIXME: cargo bench/install do not accept `--release` - if cmd != "bench" && cmd != "install" { - cargo.arg("--release"); - } - } - if self.config.locked_deps { cargo.arg("--locked"); } @@ -1853,11 +1919,14 @@ impl<'a> Builder<'a> { }; if let Some(limit) = limit { - rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={}", limit)); + if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm" + { + rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={}", limit)); + } } } - Cargo { command: cargo, rustflags, rustdocflags } + Cargo { command: cargo, rustflags, rustdocflags, allow_features } } /// Ensure that a given step is built, returning its output. This will @@ -2036,6 +2105,7 @@ pub struct Cargo { command: Command, rustflags: Rustflags, rustdocflags: Rustflags, + allow_features: String, } impl Cargo { @@ -2080,6 +2150,18 @@ impl Cargo { self.command.current_dir(dir); self } + + /// Adds nightly-only features that this invocation is allowed to use. + /// + /// By default, all nightly features are allowed. Once this is called, it + /// will be restricted to the given set. + pub fn allow_features(&mut self, features: &str) -> &mut Cargo { + if !self.allow_features.is_empty() { + self.allow_features.push(','); + } + self.allow_features.push_str(features); + self + } } impl From<Cargo> for Command { @@ -2094,6 +2176,10 @@ impl From<Cargo> for Command { cargo.command.env("RUSTDOCFLAGS", rustdocflags); } + if !cargo.allow_features.is_empty() { + cargo.command.env("RUSTC_ALLOW_FEATURES", cargo.allow_features); + } + cargo.command } } diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index 5f21d2b00..d5fcd1075 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -78,7 +78,7 @@ macro_rules! rustc { #[test] fn test_valid() { // make sure multi suite paths are accepted - check_cli(["test", "src/test/ui/attr-start.rs", "src/test/ui/attr-shebang.rs"]); + check_cli(["test", "tests/ui/attr-start.rs", "tests/ui/attr-shebang.rs"]); } #[test] diff --git a/src/bootstrap/cc_detect.rs b/src/bootstrap/cc_detect.rs index 7128d542a..65c882fb8 100644 --- a/src/bootstrap/cc_detect.rs +++ b/src/bootstrap/cc_detect.rs @@ -47,6 +47,8 @@ fn cc2ar(cc: &Path, target: TargetSelection) -> Option<PathBuf> { Some(PathBuf::from("ar")) } else if target.contains("vxworks") { Some(PathBuf::from("wr-ar")) + } else if target.contains("android") { + Some(cc.parent().unwrap().join(PathBuf::from("llvm-ar"))) } else { let parent = cc.parent().unwrap(); let file = cc.file_name().unwrap().to_str().unwrap(); @@ -219,12 +221,22 @@ fn set_compiler( } pub(crate) fn ndk_compiler(compiler: Language, triple: &str, ndk: &Path) -> PathBuf { - let triple_translated = triple - .replace("armv7neon", "arm") - .replace("armv7", "arm") - .replace("thumbv7neon", "arm") - .replace("thumbv7", "arm"); - let compiler = format!("{}-{}", triple_translated, compiler.clang()); + let mut triple_iter = triple.split("-"); + let triple_translated = if let Some(arch) = triple_iter.next() { + let arch_new = match arch { + "arm" | "armv7" | "armv7neon" | "thumbv7" | "thumbv7neon" => "armv7a", + other => other, + }; + std::iter::once(arch_new).chain(triple_iter).collect::<Vec<&str>>().join("-") + } else { + triple.to_string() + }; + + // API 19 is the earliest API level supported by NDK r25b but AArch64 and x86_64 support + // begins at API level 21. + let api_level = + if triple.contains("aarch64") || triple.contains("x86_64") { "21" } else { "19" }; + let compiler = format!("{}{}-{}", triple_translated, api_level, compiler.clang()); ndk.join("bin").join(compiler) } diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 2e1bd8d6d..4b8a58e87 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -101,7 +101,7 @@ impl Step for Std { std_cargo(builder, target, compiler.stage, &mut cargo); builder.info(&format!( - "Checking stage{} std artifacts ({} -> {})", + "Checking stage{} library artifacts ({} -> {})", builder.top_stage, &compiler.host, target )); run_cargo( @@ -111,6 +111,7 @@ impl Step for Std { &libstd_stamp(builder, compiler, target), vec![], true, + false, ); // We skip populating the sysroot in non-zero stage because that'll lead @@ -157,7 +158,7 @@ impl Step for Std { } builder.info(&format!( - "Checking stage{} std test/bench/example targets ({} -> {})", + "Checking stage{} library test/bench/example targets ({} -> {})", builder.top_stage, &compiler.host, target )); run_cargo( @@ -167,6 +168,7 @@ impl Step for Std { &libstd_test_stamp(builder, compiler, target), vec![], true, + false, ); } } @@ -243,6 +245,7 @@ impl Step for Rustc { &librustc_stamp(builder, compiler, target), vec![], true, + false, ); let libdir = builder.sysroot_libdir(compiler, target); @@ -303,6 +306,7 @@ impl Step for CodegenBackend { &codegen_backend_stamp(builder, compiler, target, backend), vec![], true, + false, ); } } @@ -342,9 +346,7 @@ impl Step for RustAnalyzer { &["rust-analyzer/in-rust-tree".to_owned()], ); - cargo.rustflag( - "-Zallow-features=proc_macro_internals,proc_macro_diagnostic,proc_macro_span", - ); + cargo.allow_features(crate::tool::RustAnalyzer::ALLOW_FEATURES); // For ./x.py clippy, don't check those targets because // linting tests and benchmarks can produce very noisy results @@ -359,7 +361,15 @@ impl Step for RustAnalyzer { "Checking stage{} {} artifacts ({} -> {})", compiler.stage, "rust-analyzer", &compiler.host.triple, target.triple )); - run_cargo(builder, cargo, args(builder), &stamp(builder, compiler, target), vec![], true); + run_cargo( + builder, + cargo, + args(builder), + &stamp(builder, compiler, target), + vec![], + true, + false, + ); /// Cargo's output path in a given stage, compiled by a particular /// compiler for the specified target. @@ -432,6 +442,7 @@ macro_rules! tool_check_step { &stamp(builder, compiler, target), vec![], true, + false, ); /// Cargo's output path in a given stage, compiled by a particular diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs index 069f3d6ac..468efc111 100644 --- a/src/bootstrap/clean.rs +++ b/src/bootstrap/clean.rs @@ -9,10 +9,81 @@ use std::fs; use std::io::{self, ErrorKind}; use std::path::Path; +use crate::builder::{crate_description, Builder, RunConfig, ShouldRun, Step}; +use crate::cache::Interned; use crate::util::t; -use crate::Build; +use crate::{Build, Compiler, Mode, Subcommand}; -pub fn clean(build: &Build, all: bool) { +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CleanAll {} + +impl Step for CleanAll { + const DEFAULT: bool = true; + type Output = (); + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CleanAll {}) + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let Subcommand::Clean { all, .. } = builder.config.cmd else { unreachable!("wrong subcommand?") }; + clean_default(builder.build, all) + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() // handled by DEFAULT + } +} + +macro_rules! clean_crate_tree { + ( $( $name:ident, $mode:path, $root_crate:literal);+ $(;)? ) => { $( + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct $name { + compiler: Compiler, + crates: Interned<Vec<String>>, + } + + impl Step for $name { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let crates = run.builder.in_tree_crates($root_crate, None); + run.crates(crates) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let compiler = builder.compiler(builder.top_stage, run.target); + builder.ensure(Self { crates: run.cargo_crates_in_set(), compiler }); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let compiler = self.compiler; + let target = compiler.host; + let mut cargo = builder.bare_cargo(compiler, $mode, target, "clean"); + for krate in &*self.crates { + cargo.arg(krate); + } + + builder.info(&format!( + "Cleaning{} stage{} {} artifacts ({} -> {})", + crate_description(&self.crates), compiler.stage, stringify!($name).to_lowercase(), &compiler.host, target, + )); + + // NOTE: doesn't use `run_cargo` because we don't want to save a stamp file, + // and doesn't use `stream_cargo` to avoid passing `--message-format` which `clean` doesn't accept. + builder.run(&mut cargo); + } + } + )+ } +} + +clean_crate_tree! { + Rustc, Mode::Rustc, "rustc-main"; + Std, Mode::Std, "test"; +} + +fn clean_default(build: &Build, all: bool) { rm_rf("tmp".as_ref()); if all { @@ -21,6 +92,7 @@ pub fn clean(build: &Build, all: bool) { rm_rf(&build.out.join("tmp")); rm_rf(&build.out.join("dist")); rm_rf(&build.out.join("bootstrap")); + rm_rf(&build.out.join("rustfmt.stamp")); for host in &build.hosts { let entries = match build.out.join(host.triple).read_dir() { diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 0deed3f99..68d1db016 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -18,6 +18,7 @@ use std::str; use serde::Deserialize; +use crate::builder::crate_description; use crate::builder::Cargo; use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; @@ -46,17 +47,6 @@ impl Std { } } -/// Return a `-p=x -p=y` string suitable for passing to a cargo invocation. -fn build_crates_in_set(run: &RunConfig<'_>) -> Interned<Vec<String>> { - let mut crates = Vec::new(); - for krate in &run.paths { - let path = krate.assert_single_path(); - let crate_name = run.builder.crate_paths[&path.path]; - crates.push(format!("-p={crate_name}")); - } - INTERNER.intern_list(crates) -} - impl Step for Std { type Output = (); const DEFAULT: bool = true; @@ -76,7 +66,7 @@ impl Step for Std { // Build all crates anyway, as if they hadn't passed the other args. let has_library = run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); - let crates = if has_library { Default::default() } else { build_crates_in_set(&run) }; + let crates = if has_library { Default::default() } else { run.cargo_crates_in_set() }; run.builder.ensure(Std { compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), target: run.target, @@ -121,7 +111,10 @@ impl Step for Std { let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); if compiler_to_use != compiler { builder.ensure(Std::new(compiler_to_use, target)); - builder.info(&format!("Uplifting stage1 std ({} -> {})", compiler_to_use.host, target)); + builder.info(&format!( + "Uplifting stage1 library ({} -> {})", + compiler_to_use.host, target + )); // Even if we're not building std this stage, the new sysroot must // still contain the third party objects needed by various targets. @@ -137,18 +130,25 @@ impl Step for Std { let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "build"); std_cargo(builder, target, compiler.stage, &mut cargo); + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } builder.info(&format!( - "Building stage{} std artifacts ({} -> {})", - compiler.stage, &compiler.host, target + "Building{} stage{} library artifacts ({} -> {})", + crate_description(&self.crates), + compiler.stage, + &compiler.host, + target, )); run_cargo( builder, cargo, - self.crates.to_vec(), + vec![], &libstd_stamp(builder, compiler, target), target_deps, false, + false, ); builder.ensure(StdLink::from_std( @@ -321,8 +321,15 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car "" }; + let mut features = String::new(); + + // Cranelift doesn't support `asm`. + if stage != 0 && builder.config.default_codegen_backend().unwrap_or_default() == "cranelift" { + features += " compiler-builtins-no-asm"; + } + if builder.no_std(target) == Some(true) { - let mut features = "compiler-builtins-mem".to_string(); + features += " compiler-builtins-mem"; if !target.starts_with("bpf") { features.push_str(compiler_builtins_c_feature); } @@ -335,7 +342,7 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car .arg("--features") .arg(features); } else { - let mut features = builder.std_features(target); + features += &builder.std_features(target); features.push_str(compiler_builtins_c_feature); cargo @@ -429,7 +436,7 @@ impl Step for StdLink { let target_compiler = self.target_compiler; let target = self.target; builder.info(&format!( - "Copying stage{} std from stage{} ({} -> {} / {})", + "Copying stage{} library from stage{} ({} -> {} / {})", target_compiler.stage, compiler.stage, &compiler.host, target_compiler.host, target )); let libdir = builder.sysroot_libdir(target_compiler, target); @@ -596,7 +603,7 @@ impl Step for Rustc { } fn make_run(run: RunConfig<'_>) { - let crates = build_crates_in_set(&run); + let crates = run.cargo_crates_in_set(); run.builder.ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), target: run.target, @@ -695,7 +702,8 @@ impl Step for Rustc { )); } - // cfg(bootstrap): remove if condition once the bootstrap compiler supports dylib LTO + // We currently don't support cross-crate LTO in stage0. This also isn't hugely necessary + // and may just be a time sink. if compiler.stage != 0 { match builder.config.rust_lto { RustcLto::Thin | RustcLto::Fat => { @@ -717,17 +725,25 @@ impl Step for Rustc { } } + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + builder.info(&format!( - "Building stage{} compiler artifacts ({} -> {})", - compiler.stage, &compiler.host, target + "Building{} stage{} compiler artifacts ({} -> {})", + crate_description(&self.crates), + compiler.stage, + &compiler.host, + target, )); run_cargo( builder, cargo, - self.crates.to_vec(), + vec![], &librustc_stamp(builder, compiler, target), vec![], false, + true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. ); builder.ensure(RustcLink::from_rustc( @@ -754,7 +770,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS .env("CFG_RELEASE_CHANNEL", &builder.config.channel) .env("CFG_VERSION", builder.rust_version()); - if let Some(backend) = builder.config.rust_codegen_backends.get(0) { + if let Some(backend) = builder.config.default_codegen_backend() { cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); } @@ -805,7 +821,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS if builder.is_rust_llvm(target) { cargo.env("LLVM_RUSTLLVM", "1"); } - let llvm_config = builder.ensure(native::Llvm { target }); + let native::LlvmResult { llvm_config, .. } = builder.ensure(native::Llvm { target }); cargo.env("LLVM_CONFIG", &llvm_config); if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { cargo.env("CFG_LLVM_ROOT", s); @@ -984,7 +1000,7 @@ impl Step for CodegenBackend { "Building stage{} codegen backend {} ({} -> {})", compiler.stage, backend, &compiler.host, target )); - let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false); + let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false); if builder.config.dry_run() { return; } @@ -1177,7 +1193,7 @@ impl Step for Sysroot { ); if builder.config.rust_remap_debuginfo { eprintln!( - "warning: some `src/test/ui` tests will fail when lacking `{}`", + "warning: some `tests/ui` tests will fail when lacking `{}`", sysroot_lib_rustlib_src_rust.display(), ); } @@ -1341,9 +1357,10 @@ impl Step for Assemble { } if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { - let llvm_config_bin = builder.ensure(native::Llvm { target: target_compiler.host }); + let native::LlvmResult { llvm_config, .. } = + builder.ensure(native::Llvm { target: target_compiler.host }); if !builder.config.dry_run() { - let llvm_bin_dir = output(Command::new(llvm_config_bin).arg("--bindir")); + let llvm_bin_dir = output(Command::new(llvm_config).arg("--bindir")); let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); // Since we've already built the LLVM tools, install them to the sysroot. @@ -1411,6 +1428,7 @@ pub fn run_cargo( stamp: &Path, additional_target_deps: Vec<(PathBuf, DependencyType)>, is_check: bool, + rlib_only_metadata: bool, ) -> Vec<PathBuf> { if builder.config.dry_run() { return Vec::new(); @@ -1444,13 +1462,35 @@ pub fn run_cargo( }; for filename in filenames { // Skip files like executables - if !(filename.ends_with(".rlib") - || filename.ends_with(".lib") + let mut keep = false; + if filename.ends_with(".lib") || filename.ends_with(".a") || is_debug_info(&filename) || is_dylib(&filename) - || (is_check && filename.ends_with(".rmeta"))) { + // Always keep native libraries, rust dylibs and debuginfo + keep = true; + } + if is_check && filename.ends_with(".rmeta") { + // During check builds we need to keep crate metadata + keep = true; + } else if rlib_only_metadata { + if filename.contains("jemalloc_sys") || filename.contains("rustc_smir") { + // jemalloc_sys and rustc_smir are not linked into librustc_driver.so, + // so we need to distribute them as rlib to be able to use them. + keep |= filename.ends_with(".rlib"); + } else { + // Distribute the rest of the rustc crates as rmeta files only to reduce + // the tarball sizes by about 50%. The object files are linked into + // librustc_driver.so, so it is still possible to link against them. + keep |= filename.ends_with(".rmeta"); + } + } else { + // In all other cases keep all rlibs + keep |= filename.ends_with(".rlib"); + } + + if !keep { continue; } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index d8c15c76e..b41d60d51 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -3,6 +3,9 @@ //! This module implements parsing `config.toml` configuration files to tweak //! how the build runs. +#[cfg(test)] +mod tests; + use std::cell::{Cell, RefCell}; use std::cmp; use std::collections::{HashMap, HashSet}; @@ -46,10 +49,7 @@ pub enum DryRun { /// Global configuration for the entire build and/or bootstrap. /// -/// This structure is derived from a combination of both `config.toml` and -/// `config.mk`. As of the time of this writing it's unlikely that `config.toml` -/// is used all that much, so this is primarily filled out by `config.mk` which -/// is generated from `./configure`. +/// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. /// /// Note that this structure is not decoded directly into, but rather it is /// filled out from the decoded forms of the structs below. For documentation @@ -635,6 +635,7 @@ define_config! { dist_stage: Option<u32> = "dist-stage", bench_stage: Option<u32> = "bench-stage", patch_binaries_for_nix: Option<bool> = "patch-binaries-for-nix", + // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally metrics: Option<bool> = "metrics", } } @@ -696,7 +697,7 @@ define_config! { } } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] #[serde(untagged)] enum StringOrBool { String(String), @@ -822,6 +823,29 @@ impl Config { } pub fn parse(args: &[String]) -> Config { + #[cfg(test)] + let get_toml = |_: &_| TomlConfig::default(); + #[cfg(not(test))] + let get_toml = |file: &Path| { + let contents = + t!(fs::read_to_string(file), format!("config file {} not found", file.display())); + // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of + // TomlConfig and sub types to be monomorphized 5x by toml. + match toml::from_str(&contents) + .and_then(|table: toml::Value| TomlConfig::deserialize(table)) + { + Ok(table) => table, + Err(err) => { + eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err); + crate::detail_exit(2); + } + } + }; + + Self::parse_inner(args, get_toml) + } + + fn parse_inner<'a>(args: &[String], get_toml: impl 'a + Fn(&Path) -> TomlConfig) -> Config { let flags = Flags::parse(&args); let mut config = Config::default_opts(); @@ -907,25 +931,6 @@ impl Config { config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json)); - #[cfg(test)] - let get_toml = |_| TomlConfig::default(); - #[cfg(not(test))] - let get_toml = |file: &Path| { - let contents = - t!(fs::read_to_string(file), format!("config file {} not found", file.display())); - // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of - // TomlConfig and sub types to be monomorphized 5x by toml. - match toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - { - Ok(table) => table, - Err(err) => { - eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err); - crate::detail_exit(2); - } - } - }; - // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. let toml_path = flags .config @@ -1063,6 +1068,79 @@ impl Config { let mut optimize = None; let mut ignore_git = None; + if let Some(rust) = toml.rust { + debug = rust.debug; + debug_assertions = rust.debug_assertions; + debug_assertions_std = rust.debug_assertions_std; + overflow_checks = rust.overflow_checks; + overflow_checks_std = rust.overflow_checks_std; + debug_logging = rust.debug_logging; + debuginfo_level = rust.debuginfo_level; + debuginfo_level_rustc = rust.debuginfo_level_rustc; + debuginfo_level_std = rust.debuginfo_level_std; + debuginfo_level_tools = rust.debuginfo_level_tools; + debuginfo_level_tests = rust.debuginfo_level_tests; + config.rust_split_debuginfo = rust + .split_debuginfo + .as_deref() + .map(SplitDebuginfo::from_str) + .map(|v| v.expect("invalid value for rust.split_debuginfo")) + .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple)); + optimize = rust.optimize; + ignore_git = rust.ignore_git; + config.rust_new_symbol_mangling = rust.new_symbol_mangling; + set(&mut config.rust_optimize_tests, rust.optimize_tests); + set(&mut config.codegen_tests, rust.codegen_tests); + set(&mut config.rust_rpath, rust.rpath); + set(&mut config.jemalloc, rust.jemalloc); + set(&mut config.test_compare_mode, rust.test_compare_mode); + set(&mut config.backtrace, rust.backtrace); + set(&mut config.channel, rust.channel); + config.description = rust.description; + set(&mut config.rust_dist_src, rust.dist_src); + set(&mut config.verbose_tests, rust.verbose_tests); + // in the case "false" is set explicitly, do not overwrite the command line args + if let Some(true) = rust.incremental { + config.incremental = true; + } + set(&mut config.use_lld, rust.use_lld); + set(&mut config.lld_enabled, rust.lld); + set(&mut config.llvm_tools_enabled, rust.llvm_tools); + config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); + config.rustc_default_linker = rust.default_linker; + config.musl_root = rust.musl_root.map(PathBuf::from); + config.save_toolstates = rust.save_toolstates.map(PathBuf::from); + set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings)); + set(&mut config.backtrace_on_ice, rust.backtrace_on_ice); + set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir); + config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit; + set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo); + set(&mut config.control_flow_guard, rust.control_flow_guard); + config.llvm_libunwind_default = rust + .llvm_libunwind + .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); + + if let Some(ref backends) = rust.codegen_backends { + config.rust_codegen_backends = + backends.iter().map(|s| INTERNER.intern_str(s)).collect(); + } + + config.rust_codegen_units = rust.codegen_units.map(threads_from_config); + config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); + config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); + config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); + config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); + + config.rust_lto = rust + .lto + .as_deref() + .map(|value| RustcLto::from_str(value).unwrap()) + .unwrap_or_default(); + } else { + config.rust_profile_use = flags.rust_profile_use; + config.rust_profile_generate = flags.rust_profile_generate; + } + if let Some(llvm) = toml.llvm { match llvm.ccache { Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), @@ -1099,13 +1177,17 @@ impl Config { config.llvm_polly = llvm.polly.unwrap_or(false); config.llvm_clang = llvm.clang.unwrap_or(false); config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default()); + + let asserts = llvm_assertions.unwrap_or(false); config.llvm_from_ci = match llvm.download_ci_llvm { Some(StringOrBool::String(s)) => { assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s); - crate::native::is_ci_llvm_available(&config, llvm_assertions.unwrap_or(false)) + crate::native::is_ci_llvm_available(&config, asserts) } Some(StringOrBool::Bool(b)) => b, - None => false, + None => { + config.channel == "dev" && crate::native::is_ci_llvm_available(&config, asserts) + } }; if config.llvm_from_ci { @@ -1145,79 +1227,9 @@ impl Config { // the link step) with each stage. config.llvm_link_shared.set(Some(true)); } - } - - if let Some(rust) = toml.rust { - debug = rust.debug; - debug_assertions = rust.debug_assertions; - debug_assertions_std = rust.debug_assertions_std; - overflow_checks = rust.overflow_checks; - overflow_checks_std = rust.overflow_checks_std; - debug_logging = rust.debug_logging; - debuginfo_level = rust.debuginfo_level; - debuginfo_level_rustc = rust.debuginfo_level_rustc; - debuginfo_level_std = rust.debuginfo_level_std; - debuginfo_level_tools = rust.debuginfo_level_tools; - debuginfo_level_tests = rust.debuginfo_level_tests; - config.rust_split_debuginfo = rust - .split_debuginfo - .as_deref() - .map(SplitDebuginfo::from_str) - .map(|v| v.expect("invalid value for rust.split_debuginfo")) - .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple)); - optimize = rust.optimize; - ignore_git = rust.ignore_git; - config.rust_new_symbol_mangling = rust.new_symbol_mangling; - set(&mut config.rust_optimize_tests, rust.optimize_tests); - set(&mut config.codegen_tests, rust.codegen_tests); - set(&mut config.rust_rpath, rust.rpath); - set(&mut config.jemalloc, rust.jemalloc); - set(&mut config.test_compare_mode, rust.test_compare_mode); - set(&mut config.backtrace, rust.backtrace); - set(&mut config.channel, rust.channel); - config.description = rust.description; - set(&mut config.rust_dist_src, rust.dist_src); - set(&mut config.verbose_tests, rust.verbose_tests); - // in the case "false" is set explicitly, do not overwrite the command line args - if let Some(true) = rust.incremental { - config.incremental = true; - } - set(&mut config.use_lld, rust.use_lld); - set(&mut config.lld_enabled, rust.lld); - set(&mut config.llvm_tools_enabled, rust.llvm_tools); - config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); - config.rustc_default_linker = rust.default_linker; - config.musl_root = rust.musl_root.map(PathBuf::from); - config.save_toolstates = rust.save_toolstates.map(PathBuf::from); - set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings)); - set(&mut config.backtrace_on_ice, rust.backtrace_on_ice); - set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir); - config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit; - set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo); - set(&mut config.control_flow_guard, rust.control_flow_guard); - config.llvm_libunwind_default = rust - .llvm_libunwind - .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); - - if let Some(ref backends) = rust.codegen_backends { - config.rust_codegen_backends = - backends.iter().map(|s| INTERNER.intern_str(s)).collect(); - } - - config.rust_codegen_units = rust.codegen_units.map(threads_from_config); - config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); - config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); - config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); - config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); - - config.rust_lto = rust - .lto - .as_deref() - .map(|value| RustcLto::from_str(value).unwrap()) - .unwrap_or_default(); } else { - config.rust_profile_use = flags.rust_profile_use; - config.rust_profile_generate = flags.rust_profile_generate; + config.llvm_from_ci = + config.channel == "dev" && crate::native::is_ci_llvm_available(&config, false); } if let Some(t) = toml.target { @@ -1596,6 +1608,10 @@ impl Config { self.submodules.unwrap_or(rust_info.is_managed_git_subrepository()) } + pub fn default_codegen_backend(&self) -> Option<Interned<String>> { + self.rust_codegen_backends.get(0).cloned() + } + /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. fn download_ci_rustc_commit(&self, download_rustc: Option<StringOrBool>) -> Option<String> { // If `download-rustc` is not set, default to rebuilding. diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/config/tests.rs new file mode 100644 index 000000000..c30c91317 --- /dev/null +++ b/src/bootstrap/config/tests.rs @@ -0,0 +1,24 @@ +use super::{Config, TomlConfig}; +use std::path::Path; + +fn toml(config: &str) -> impl '_ + Fn(&Path) -> TomlConfig { + |&_| toml::from_str(config).unwrap() +} + +fn parse(config: &str) -> Config { + Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], toml(config)) +} + +#[test] +fn download_ci_llvm() { + let parse_llvm = |s| parse(s).llvm_from_ci; + let if_available = parse_llvm("llvm.download-ci-llvm = \"if-available\""); + + assert!(parse_llvm("llvm.download-ci-llvm = true")); + assert!(!parse_llvm("llvm.download-ci-llvm = false")); + assert_eq!(parse_llvm(""), if_available); + assert_eq!(parse_llvm("rust.channel = \"dev\""), if_available); + assert!(!parse_llvm("rust.channel = \"stable\"")); +} + +// FIXME: add test for detecting `src` and `out` diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 6b139decb..0af329e70 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -45,7 +45,7 @@ o("llvm-static-stdcpp", "llvm.static-libstdcpp", "statically link to libstdc++ f o("llvm-link-shared", "llvm.link-shared", "prefer shared linking to LLVM (llvm-config --link-shared)") o("rpath", "rust.rpath", "build rpaths into rustc itself") o("llvm-version-check", "llvm.version-check", "check if the LLVM version is supported, build anyway") -o("codegen-tests", "rust.codegen-tests", "run the src/test/codegen tests") +o("codegen-tests", "rust.codegen-tests", "run the tests/codegen tests") o("option-checking", None, "complain about unrecognized options in this configure script") o("ninja", "llvm.ninja", "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)") o("locked-deps", "build.locked-deps", "force Cargo.lock to be up to date") @@ -405,7 +405,9 @@ if 'target' in config: configured_targets.append(target) for target in configured_targets: targets[target] = sections['target'][:] - targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target)) + # For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target. + # Avoid using quotes unless it's necessary. + targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target) if "." in target else target) def is_number(value): diff --git a/src/bootstrap/defaults/config.user.toml b/src/bootstrap/defaults/config.user.toml index 6647061d8..48ae2fe44 100644 --- a/src/bootstrap/defaults/config.user.toml +++ b/src/bootstrap/defaults/config.user.toml @@ -7,3 +7,7 @@ test-stage = 2 doc-stage = 2 # When compiling from source, you usually want all tools. extended = true + +[llvm] +# Most users installing from source want to build all parts of the project from source, not just rustc itself. +download-ci-llvm = false diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 3cb0eccd3..6594b23c5 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -24,6 +24,7 @@ use crate::channel; use crate::compile; use crate::config::TargetSelection; use crate::doc::DocumentationFormat; +use crate::native; use crate::tarball::{GeneratedTarball, OverlayKind, Tarball}; use crate::tool::{self, Tool}; use crate::util::{exe, is_dylib, output, t, timeit}; @@ -951,7 +952,7 @@ impl Step for PlainSourceTarball { "Cargo.toml", "Cargo.lock", ]; - let src_dirs = ["src", "compiler", "library"]; + let src_dirs = ["src", "compiler", "library", "tests"]; copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src); @@ -1129,12 +1130,6 @@ impl Step for RustAnalyzer { let compiler = self.compiler; let target = self.target; - if target.contains("riscv64") { - // riscv64 currently has an LLVM bug that makes rust-analyzer unable - // to build. See #74813 for details. - return None; - } - let rust_analyzer = builder .ensure(tool::RustAnalyzer { compiler, target }) .expect("rust-analyzer always builds"); @@ -1927,7 +1922,9 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir builder.install(&llvm_dylib_path, dst_libdir, 0o644); } !builder.config.dry_run() - } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) { + } else if let Ok(native::LlvmResult { llvm_config, .. }) = + native::prebuilt_llvm_config(builder, target) + { let mut cmd = Command::new(llvm_config); cmd.arg("--libfiles"); builder.verbose(&format!("running {:?}", cmd)); @@ -2064,6 +2061,9 @@ impl Step for RustDev { builder.ensure(crate::native::Llvm { target }); + // We want to package `lld` to use it with `download-ci-llvm`. + builder.ensure(crate::native::Lld { target }); + let src_bindir = builder.llvm_out(target).join("bin"); // If updating this list, you likely want to change // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users @@ -2137,7 +2137,7 @@ impl Step for Bootstrap { let tarball = Tarball::new(builder, "bootstrap", &target.triple); let bootstrap_outdir = &builder.bootstrap_out; - for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] { + for file in &["bootstrap", "rustc", "rustdoc", "sccache-plus-cl"] { tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755); } diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 2c6fd1e1d..9bad9046e 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -12,6 +12,7 @@ use std::fs; use std::io; use std::path::{Path, PathBuf}; +use crate::builder::crate_description; use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; use crate::compile; @@ -506,7 +507,11 @@ impl Step for Std { // Look for library/std, library/core etc in the `x.py doc` arguments and // open the corresponding rendered docs. for requested_crate in requested_crates { - if STD_PUBLIC_CRATES.iter().any(|k| *k == requested_crate.as_str()) { + if requested_crate == "library" { + // For `x.py doc library --open`, open `std` by default. + let index = out.join("std").join("index.html"); + builder.open_in_browser(index); + } else if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) { let index = out.join(requested_crate).join("index.html"); builder.open_in_browser(index); } @@ -554,7 +559,8 @@ fn doc_std( requested_crates: &[String], ) { builder.info(&format!( - "Documenting stage{} std ({}) in {} format", + "Documenting{} stage{} library ({}) in {} format", + crate_description(requested_crates), stage, target, format.as_str() @@ -729,7 +735,7 @@ impl Step for Rustc { } macro_rules! tool_doc { - ($tool: ident, $should_run: literal, $path: literal, [$($krate: literal),+ $(,)?] $(,)?) => { + ($tool: ident, $should_run: literal, $path: literal, $(rustc_tool = $rustc_tool:literal, )? $(in_tree = $in_tree:literal, )? [$($krate: literal),+ $(,)?] $(,)?) => { #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct $tool { target: TargetSelection, @@ -763,13 +769,24 @@ macro_rules! tool_doc { let out = builder.compiler_doc_out(target); t!(fs::create_dir_all(&out)); - // Build rustc docs so that we generate relative links. - builder.ensure(Rustc { stage, target }); - // Rustdoc needs the rustc sysroot available to build. - // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed - // with strange errors, but only on a full bors test ... let compiler = builder.compiler(stage, builder.config.build); - builder.ensure(compile::Rustc::new(compiler, target)); + builder.ensure(compile::Std::new(compiler, target)); + + if true $(&& $rustc_tool)? { + // Build rustc docs so that we generate relative links. + builder.ensure(Rustc { stage, target }); + + // Rustdoc needs the rustc sysroot available to build. + // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed + // with strange errors, but only on a full bors test ... + builder.ensure(compile::Rustc::new(compiler, target)); + } + + let source_type = if true $(&& $in_tree)? { + SourceType::InTree + } else { + SourceType::Submodule + }; builder.info( &format!( @@ -781,9 +798,15 @@ macro_rules! tool_doc { ); // Symlink compiler docs to the output directory of rustdoc documentation. - let out_dir = builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"); - t!(fs::create_dir_all(&out_dir)); - t!(symlink_dir_force(&builder.config, &out, &out_dir)); + let out_dirs = [ + builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"), + // Cargo uses a different directory for proc macros. + builder.stage_out(compiler, Mode::ToolRustc).join("doc"), + ]; + for out_dir in out_dirs { + t!(fs::create_dir_all(&out_dir)); + t!(symlink_dir_force(&builder.config, &out, &out_dir)); + } // Build cargo command. let mut cargo = prepare_tool_cargo( @@ -793,7 +816,7 @@ macro_rules! tool_doc { target, "doc", $path, - SourceType::InTree, + source_type, &[], ); @@ -824,6 +847,30 @@ tool_doc!( ); tool_doc!(Clippy, "clippy", "src/tools/clippy", ["clippy_utils"]); tool_doc!(Miri, "miri", "src/tools/miri", ["miri"]); +tool_doc!( + Cargo, + "cargo", + "src/tools/cargo", + rustc_tool = false, + in_tree = false, + [ + "cargo", + "cargo-platform", + "cargo-util", + "crates-io", + "cargo-test-macro", + "cargo-test-support", + "cargo-credential", + "cargo-credential-1password", + "mdman", + // FIXME: this trips a license check in tidy. + // "resolver-tests", + // FIXME: we should probably document these, but they're different per-platform so we can't use `tool_doc`. + // "cargo-credential-gnome-secret", + // "cargo-credential-macos-keychain", + // "cargo-credential-wincred", + ] +); #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct ErrorIndex { diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp index d19a1ae95..94630e40f 100644 --- a/src/bootstrap/download-ci-llvm-stamp +++ b/src/bootstrap/download-ci-llvm-stamp @@ -1,4 +1,4 @@ Change this file to make users of the `download-ci-llvm` configuration download a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. -Last change is for: https://github.com/rust-lang/rust/pull/102790 +Last change is for: https://github.com/rust-lang/rust/pull/104748 diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 37a8eb884..52c3dc0bf 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -130,6 +130,7 @@ pub enum Subcommand { test_args: Vec<String>, }, Clean { + paths: Vec<PathBuf>, all: bool, }, Dist { @@ -143,7 +144,7 @@ pub enum Subcommand { args: Vec<String>, }, Setup { - profile: Option<Profile>, + profile: Option<PathBuf>, }, } @@ -351,22 +352,32 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", // fn usage() let usage = |exit_code: i32, opts: &Options, verbose: bool, subcommand_help: &str| -> ! { - let config = Config::parse(&["build".to_string()]); - let build = Build::new(config); - let paths = Builder::get_help(&build, subcommand); - println!("{}", opts.usage(subcommand_help)); - if let Some(s) = paths { - if verbose { + if verbose { + // We have an unfortunate situation here: some Steps use `builder.in_tree_crates` to determine their paths. + // To determine those crates, we need to run `cargo metadata`, which means we need all submodules to be checked out. + // That takes a while to run, so only do it when paths were explicitly requested, not on all CLI errors. + // `Build::new` won't load submodules for the `setup` command. + let cmd = if verbose { + println!("note: updating submodules before printing available paths"); + "build" + } else { + "setup" + }; + let config = Config::parse(&[cmd.to_string()]); + let build = Build::new(config); + let paths = Builder::get_help(&build, subcommand); + + if let Some(s) = paths { println!("{}", s); } else { - println!( - "Run `./x.py {} -h -v` to see a list of available paths.", - subcommand.as_str() - ); + panic!("No paths available for subcommand `{}`", subcommand.as_str()); } - } else if verbose { - panic!("No paths available for subcommand `{}`", subcommand.as_str()); + } else { + println!( + "Run `./x.py {} -h -v` to see a list of available paths.", + subcommand.as_str() + ); } crate::detail_exit(exit_code); }; @@ -476,13 +487,13 @@ Arguments: This subcommand accepts a number of paths to test directories that should be compiled and run. For example: - ./x.py test src/test/ui + ./x.py test tests/ui ./x.py test library/std --test-args hash_map ./x.py test library/std --stage 0 --no-doc - ./x.py test src/test/ui --bless - ./x.py test src/test/ui --compare-mode chalk + ./x.py test tests/ui --bless + ./x.py test tests/ui --compare-mode chalk - Note that `test src/test/* --stage N` does NOT depend on `build compiler/rustc --stage N`; + Note that `test tests/* --stage N` does NOT depend on `build compiler/rustc --stage N`; just like `build library/std --stage N` it tests the compiler produced by the previous stage. @@ -601,14 +612,7 @@ Arguments: open: matches.opt_present("open"), json: matches.opt_present("json"), }, - Kind::Clean => { - if !paths.is_empty() { - println!("\nclean does not take a path argument\n"); - usage(1, &opts, verbose, &subcommand_help); - } - - Subcommand::Clean { all: matches.opt_present("all") } - } + Kind::Clean => Subcommand::Clean { all: matches.opt_present("all"), paths }, Kind::Format => Subcommand::Format { check: matches.opt_present("check"), paths }, Kind::Dist => Subcommand::Dist { paths }, Kind::Install => Subcommand::Install { paths }, @@ -621,7 +625,7 @@ Arguments: } Kind::Setup => { let profile = if paths.len() > 1 { - println!("\nat most one profile can be passed to setup\n"); + eprintln!("\nerror: At most one profile can be passed to setup\n"); usage(1, &opts, verbose, &subcommand_help) } else if let Some(path) = paths.pop() { let profile_string = t!(path.into_os_string().into_string().map_err( diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs index 5e7264fe7..bfc57a85c 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/format.rs @@ -1,14 +1,15 @@ //! Runs rustfmt on the repository. use crate::builder::Builder; -use crate::util::{output, t}; +use crate::util::{output, output_result, program_out_of_date, t}; +use build_helper::git::updated_master_branch; use ignore::WalkBuilder; use std::collections::VecDeque; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::mpsc::SyncSender; -fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut() { +fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut(bool) -> bool { let mut cmd = Command::new(&rustfmt); // avoid the submodule config paths from coming into play, // we only allow a single global config for the workspace for now @@ -23,7 +24,13 @@ fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl F let cmd_debug = format!("{:?}", cmd); let mut cmd = cmd.spawn().expect("running rustfmt"); // poor man's async: return a closure that'll wait for rustfmt's completion - move || { + move |block: bool| -> bool { + if !block { + match cmd.try_wait() { + Ok(Some(_)) => {} + _ => return false, + } + } let status = cmd.wait().unwrap(); if !status.success() { eprintln!( @@ -34,7 +41,62 @@ fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl F ); crate::detail_exit(1); } + true + } +} + +fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, PathBuf)> { + let stamp_file = build.out.join("rustfmt.stamp"); + + let mut cmd = Command::new(match build.initial_rustfmt() { + Some(p) => p, + None => return None, + }); + cmd.arg("--version"); + let output = match cmd.output() { + Ok(status) => status, + Err(_) => return None, + }; + if !output.status.success() { + return None; + } + Some((String::from_utf8(output.stdout).unwrap(), stamp_file)) +} + +/// Return whether the format cache can be reused. +fn verify_rustfmt_version(build: &Builder<'_>) -> bool { + let Some((version, stamp_file)) = get_rustfmt_version(build) else {return false;}; + !program_out_of_date(&stamp_file, &version) +} + +/// Updates the last rustfmt version used +fn update_rustfmt_version(build: &Builder<'_>) { + let Some((version, stamp_file)) = get_rustfmt_version(build) else {return;}; + t!(std::fs::write(stamp_file, version)) +} + +/// Returns the Rust files modified between the `merge-base` of HEAD and +/// rust-lang/master and what is now on the disk. +/// +/// Returns `None` if all files should be formatted. +fn get_modified_rs_files(build: &Builder<'_>) -> Result<Option<Vec<String>>, String> { + let Ok(updated_master) = updated_master_branch(Some(&build.config.src)) else { return Ok(None); }; + + if !verify_rustfmt_version(build) { + return Ok(None); } + + let merge_base = + output_result(build.config.git().arg("merge-base").arg(&updated_master).arg("HEAD"))?; + Ok(Some( + output_result( + build.config.git().arg("diff-index").arg("--name-only").arg(merge_base.trim()), + )? + .lines() + .map(|s| s.trim().to_owned()) + .filter(|f| Path::new(f).extension().map_or(false, |ext| ext == "rs")) + .collect(), + )) } #[derive(serde::Deserialize)] @@ -71,6 +133,9 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { Ok(status) => status.success(), Err(_) => false, }; + + let mut paths = paths.to_vec(); + if git_available { let in_working_tree = match build .config @@ -103,12 +168,32 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { // preventing the latter from being formatted. ignore_fmt.add(&format!("!/{}", untracked_path)).expect(&untracked_path); } + if !check && paths.is_empty() { + match get_modified_rs_files(build) { + Ok(Some(files)) => { + for file in files { + println!("formatting modified file {file}"); + ignore_fmt.add(&format!("/{file}")).expect(&file); + } + } + Ok(None) => {} + Err(err) => { + println!( + "WARN: Something went wrong when running git commands:\n{err}\n\ + Falling back to formatting all files." + ); + // Something went wrong when getting the version. Just format all the files. + paths.push(".".into()); + } + } + } } else { println!("Not in git tree. Skipping git-aware format checks"); } } else { println!("Could not find usable git. Skipping git-aware format checks"); } + let ignore_fmt = ignore_fmt.build().unwrap(); let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { @@ -146,15 +231,23 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check); children.push_back(child); + // poll completion before waiting + for i in (0..children.len()).rev() { + if children[i](false) { + children.swap_remove_back(i); + break; + } + } + if children.len() >= max_processes { // await oldest child - children.pop_front().unwrap()(); + children.pop_front().unwrap()(true); } } // await remaining children for mut child in children { - child(); + child(true); } }); @@ -172,4 +265,7 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { drop(tx); thread.join().unwrap(); + if !check { + update_rustfmt_version(build); + } } diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index 38426f3a4..1815a0973 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -209,6 +209,12 @@ install!((self, builder, _config), ); } }; + LlvmTools, alias = "llvm-tools", Self::should_build(_config), only_hosts: true, { + let tarball = builder + .ensure(dist::LlvmTools { target: self.target }) + .expect("missing llvm-tools"); + install_sh(builder, "llvm-tools", self.compiler.stage, Some(self.target), &tarball); + }; Rustfmt, alias = "rustfmt", Self::should_build(_config), only_hosts: true, { if let Some(tarball) = builder.ensure(dist::Rustfmt { compiler: self.compiler, diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 3ed534523..0474ab344 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -108,10 +108,12 @@ use std::collections::{HashMap, HashSet}; use std::env; use std::fs::{self, File}; use std::io; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::process::Command; use std::str; +use build_helper::ci::CiEnv; use channel::GitInfo; use config::{DryRun, Target}; use filetime::FileTime; @@ -119,7 +121,9 @@ use once_cell::sync::OnceCell; use crate::builder::Kind; use crate::config::{LlvmLibunwind, TargetSelection}; -use crate::util::{exe, libdir, mtime, output, run, run_suppressed, try_run_suppressed, CiEnv}; +use crate::util::{ + exe, libdir, mtime, output, run, run_suppressed, symlink_dir, try_run_suppressed, +}; mod bolt; mod builder; @@ -199,6 +203,7 @@ const EXTRA_CHECK_CFGS: &[(Option<Mode>, &'static str, Option<&[&'static str]>)] (None, "bootstrap", None), (Some(Mode::Rustc), "parallel_compiler", None), (Some(Mode::ToolRustc), "parallel_compiler", None), + (Some(Mode::ToolRustc), "emulate_second_only_system", None), (Some(Mode::Codegen), "parallel_compiler", None), (Some(Mode::Std), "stdarch_intel_sde", None), (Some(Mode::Std), "no_fp_fmt_parse", None), @@ -586,6 +591,21 @@ impl Build { metadata::build(&mut build); } + // Make a symbolic link so we can use a consistent directory in the documentation. + let build_triple = build.out.join(&build.build.triple); + t!(fs::create_dir_all(&build_triple)); + let host = build.out.join("host"); + if let Err(e) = symlink_dir(&build.config, &build_triple, &host) { + if e.kind() != ErrorKind::AlreadyExists { + panic!( + "symlink_dir({} => {}) failed with {}", + host.display(), + build_triple.display(), + e + ); + } + } + build } @@ -709,14 +729,6 @@ impl Build { return format::format(&builder::Builder::new(&self), *check, &paths); } - if let Subcommand::Clean { all } = self.config.cmd { - return clean::clean(self, all); - } - - if let Subcommand::Setup { profile } = &self.config.cmd { - return setup::setup(&self.config, *profile); - } - // Download rustfmt early so that it can be used in rust-analyzer configs. let _ = &builder::Builder::new(&self).initial_rustfmt(); @@ -782,7 +794,7 @@ impl Build { /// Gets the space-separated set of activated features for the standard /// library. fn std_features(&self, target: TargetSelection) -> String { - let mut features = "panic-unwind".to_string(); + let mut features = " panic-unwind".to_string(); match self.config.llvm_libunwind(target) { LlvmLibunwind::InTree => features.push_str(" llvm-libunwind"), @@ -1386,7 +1398,10 @@ impl Build { let mut list = vec![INTERNER.intern_str(root)]; let mut visited = HashSet::new(); while let Some(krate) = list.pop() { - let krate = &self.crates[&krate]; + let krate = self + .crates + .get(&krate) + .unwrap_or_else(|| panic!("metadata missing for {krate}: {:?}", self.crates)); ret.push(krate); for dep in &krate.deps { if !self.crates.contains_key(dep) { @@ -1639,10 +1654,10 @@ fn chmod(_path: &Path, _perms: u32) {} /// If the test is running and code is an error code, it will cause a panic. fn detail_exit(code: i32) -> ! { // if in test and code is an error code, panic with status code provided - if cfg!(test) && code != 0 { + if cfg!(test) { panic!("status code: {}", code); } else { - //otherwise,exit with provided status code + // otherwise,exit with provided status code std::process::exit(code); } } diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 9a08a7be0..d54a21b9f 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -57,13 +57,8 @@ tidy: prepare: $(Q)$(BOOTSTRAP) build --stage 2 nonexistent/path/to/trigger/cargo/metadata -check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu: - $(Q)$(BOOTSTRAP) test --stage 2 --target arm-linux-androideabi -check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu: - $(Q)$(BOOTSTRAP) test --stage 2 --target x86_64-unknown-linux-musl - TESTS_IN_2 := \ - src/test/ui \ + tests/ui \ src/tools/linkchecker ## MSVC native builders @@ -77,7 +72,7 @@ ci-subset-2: ## MingW native builders TESTS_IN_MINGW_2 := \ - src/test/ui + tests/ui ci-mingw-subset-1: $(Q)$(CFG_SRC_DIR)/x test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %) diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index f6c453ebe..e0d1504c9 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -24,9 +24,20 @@ use crate::util::get_clang_cl_resource_dir; use crate::util::{self, exe, output, t, up_to_date}; use crate::{CLang, GitRepo}; +use build_helper::ci::CiEnv; + +#[derive(Clone)] +pub struct LlvmResult { + /// Path to llvm-config binary. + /// NB: This is always the host llvm-config! + pub llvm_config: PathBuf, + /// Path to LLVM cmake directory for the target. + pub llvm_cmake_dir: PathBuf, +} + pub struct Meta { stamp: HashStamp, - build_llvm_config: PathBuf, + res: LlvmResult, out_dir: PathBuf, root: String, } @@ -54,17 +65,17 @@ impl LdFlags { } } -// This returns whether we've already previously built LLVM. -// -// It's used to avoid busting caches during x.py check -- if we've already built -// LLVM, it's fine for us to not try to avoid doing so. -// -// This will return the llvm-config if it can get it (but it will not build it -// if not). +/// This returns whether we've already previously built LLVM. +/// +/// It's used to avoid busting caches during x.py check -- if we've already built +/// LLVM, it's fine for us to not try to avoid doing so. +/// +/// This will return the llvm-config if it can get it (but it will not build it +/// if not). pub fn prebuilt_llvm_config( builder: &Builder<'_>, target: TargetSelection, -) -> Result<PathBuf, Meta> { +) -> Result<LlvmResult, Meta> { builder.config.maybe_download_ci_llvm(); // If we're using a custom LLVM bail out here, but we can only use a @@ -72,7 +83,14 @@ pub fn prebuilt_llvm_config( if let Some(config) = builder.config.target_config.get(&target) { if let Some(ref s) = config.llvm_config { check_llvm_version(builder, s); - return Ok(s.to_path_buf()); + let llvm_config = s.to_path_buf(); + let mut llvm_cmake_dir = llvm_config.clone(); + llvm_cmake_dir.pop(); + llvm_cmake_dir.pop(); + llvm_cmake_dir.push("lib"); + llvm_cmake_dir.push("cmake"); + llvm_cmake_dir.push("llvm"); + return Ok(LlvmResult { llvm_config, llvm_cmake_dir }); } } @@ -84,8 +102,9 @@ pub fn prebuilt_llvm_config( llvm_config_ret_dir.push("build"); } llvm_config_ret_dir.push("bin"); - let build_llvm_config = llvm_config_ret_dir.join(exe("llvm-config", builder.config.build)); + let llvm_cmake_dir = out_dir.join("lib/cmake/llvm"); + let res = LlvmResult { llvm_config: build_llvm_config, llvm_cmake_dir }; let stamp = out_dir.join("llvm-finished-building"); let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); @@ -96,7 +115,7 @@ pub fn prebuilt_llvm_config( Using a potentially stale build of LLVM; \ This may not behave well.", ); - return Ok(build_llvm_config); + return Ok(res); } if stamp.is_done() { @@ -110,10 +129,10 @@ pub fn prebuilt_llvm_config( stamp.path.display() )); } - return Ok(build_llvm_config); + return Ok(res); } - Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) + Err(Meta { stamp, res, out_dir, root: root.into() }) } /// This retrieves the LLVM sha we *want* to use, according to git history. @@ -200,7 +219,7 @@ pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { return false; } - if crate::util::CiEnv::is_ci() { + if CiEnv::is_ci() { // We assume we have access to git, so it's okay to unconditionally pass // `true` here. let llvm_sha = detect_llvm_sha(config, true); @@ -223,7 +242,7 @@ pub struct Llvm { } impl Step for Llvm { - type Output = PathBuf; // path to llvm-config + type Output = LlvmResult; const ONLY_HOSTS: bool = true; @@ -236,7 +255,7 @@ impl Step for Llvm { } /// Compile LLVM for `target`. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> LlvmResult { let target = self.target; let target_native = if self.target.starts_with("riscv") { // RISC-V target triples in Rust is not named the same as C compiler target triples. @@ -252,11 +271,10 @@ impl Step for Llvm { target.to_string() }; - let Meta { stamp, build_llvm_config, out_dir, root } = - match prebuilt_llvm_config(builder, target) { - Ok(p) => return p, - Err(m) => m, - }; + let Meta { stamp, res, out_dir, root } = match prebuilt_llvm_config(builder, target) { + Ok(p) => return p, + Err(m) => m, + }; builder.update_submodule(&Path::new("src").join("llvm-project")); if builder.llvm_link_shared() && target.contains("windows") { @@ -430,7 +448,8 @@ impl Step for Llvm { // https://llvm.org/docs/HowToCrossCompileLLVM.html if target != builder.config.build { - let llvm_config = builder.ensure(Llvm { target: builder.config.build }); + let LlvmResult { llvm_config, .. } = + builder.ensure(Llvm { target: builder.config.build }); if !builder.config.dry_run() { let llvm_bindir = output(Command::new(&llvm_config).arg("--bindir")); let host_bin = Path::new(llvm_bindir.trim()); @@ -480,7 +499,7 @@ impl Step for Llvm { // tools and libs on all platforms. if builder.config.dry_run() { - return build_llvm_config; + return res; } cfg.build(); @@ -490,7 +509,7 @@ impl Step for Llvm { // for a versioned path like libLLVM-14.dylib. Manually create a symbolic // link to make llvm-config happy. if builder.llvm_link_shared() && target.contains("apple-darwin") { - let mut cmd = Command::new(&build_llvm_config); + let mut cmd = Command::new(&res.llvm_config); let version = output(cmd.arg("--version")); let major = version.split('.').next().unwrap(); let lib_name = match llvm_version_suffix { @@ -509,18 +528,18 @@ impl Step for Llvm { // LLVM after a configuration change, so to rebuild it the build files have to be removed, // which will also remove these modified files. if builder.config.llvm_bolt_profile_generate { - instrument_with_bolt_inplace(&get_built_llvm_lib_path(&build_llvm_config)); + instrument_with_bolt_inplace(&get_built_llvm_lib_path(&res.llvm_config)); } if let Some(path) = &builder.config.llvm_bolt_profile_use { optimize_library_with_bolt_inplace( - &get_built_llvm_lib_path(&build_llvm_config), + &get_built_llvm_lib_path(&res.llvm_config), &Path::new(path), ); } t!(stamp.write()); - build_llvm_config + res } } @@ -600,6 +619,9 @@ fn configure_cmake( if target.starts_with("aarch64") { // macOS uses a different name for building arm64 cfg.define("CMAKE_OSX_ARCHITECTURES", "arm64"); + } else if target.starts_with("i686") { + // macOS uses a different name for building i386 + cfg.define("CMAKE_OSX_ARCHITECTURES", "i386"); } else { cfg.define("CMAKE_OSX_ARCHITECTURES", target.triple.split('-').next().unwrap()); } @@ -803,7 +825,21 @@ impl Step for Lld { } let target = self.target; - let llvm_config = builder.ensure(Llvm { target: self.target }); + let LlvmResult { llvm_config, llvm_cmake_dir } = builder.ensure(Llvm { target }); + + // The `dist` step packages LLD next to LLVM's binaries for download-ci-llvm. The root path + // we usually expect here is `./build/$triple/ci-llvm/`, with the binaries in its `bin` + // subfolder. We check if that's the case, and if LLD's binary already exists there next to + // `llvm-config`: if so, we can use it instead of building LLVM/LLD from source. + let ci_llvm_bin = llvm_config.parent().unwrap(); + if ci_llvm_bin.is_dir() && ci_llvm_bin.file_name().unwrap() == "bin" { + let lld_path = ci_llvm_bin.join(exe("lld", target)); + if lld_path.exists() { + // The following steps copying `lld` as `rust-lld` to the sysroot, expect it in the + // `bin` subfolder of this step's out dir. + return ci_llvm_bin.parent().unwrap().to_path_buf(); + } + } let out_dir = builder.lld_out(target); let done_stamp = out_dir.join("lld-finished-building"); @@ -834,22 +870,6 @@ impl Step for Lld { configure_cmake(builder, target, &mut cfg, true, ldflags); configure_llvm(builder, target, &mut cfg); - // This is an awful, awful hack. Discovered when we migrated to using - // clang-cl to compile LLVM/LLD it turns out that LLD, when built out of - // tree, will execute `llvm-config --cmakedir` and then tell CMake about - // that directory for later processing. Unfortunately if this path has - // forward slashes in it (which it basically always does on Windows) - // then CMake will hit a syntax error later on as... something isn't - // escaped it seems? - // - // Instead of attempting to fix this problem in upstream CMake and/or - // LLVM/LLD we just hack around it here. This thin wrapper will take the - // output from llvm-config and replace all instances of `\` with `/` to - // ensure we don't hit the same bugs with escaping. It means that you - // can't build on a system where your paths require `\` on Windows, but - // there's probably a lot of reasons you can't do that other than this. - let llvm_config_shim = env::current_exe().unwrap().with_file_name("llvm-config-wrapper"); - // Re-use the same flags as llvm to control the level of debug information // generated for lld. let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { @@ -860,36 +880,17 @@ impl Step for Lld { cfg.out_dir(&out_dir) .profile(profile) - .env("LLVM_CONFIG_REAL", &llvm_config) - .define("LLVM_CONFIG_PATH", llvm_config_shim) + .define("LLVM_CMAKE_DIR", llvm_cmake_dir) .define("LLVM_INCLUDE_TESTS", "OFF"); - // While we're using this horrible workaround to shim the execution of - // llvm-config, let's just pile on more. I can't seem to figure out how - // to build LLD as a standalone project and also cross-compile it at the - // same time. It wants a natively executable `llvm-config` to learn - // about LLVM, but then it learns about all the host configuration of - // LLVM and tries to link to host LLVM libraries. - // - // To work around that we tell our shim to replace anything with the - // build target with the actual target instead. This'll break parts of - // LLD though which try to execute host tools, such as llvm-tblgen, so - // we specifically tell it where to find those. This is likely super - // brittle and will break over time. If anyone knows better how to - // cross-compile LLD it would be much appreciated to fix this! if target != builder.config.build { - cfg.env("LLVM_CONFIG_SHIM_REPLACE", &builder.config.build.triple) - .env("LLVM_CONFIG_SHIM_REPLACE_WITH", &target.triple) - .define( - "LLVM_TABLEGEN_EXE", - llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION), - ); + // Use the host llvm-tblgen binary. + cfg.define( + "LLVM_TABLEGEN_EXE", + llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION), + ); } - // Explicitly set C++ standard, because upstream doesn't do so - // for standalone builds. - cfg.define("CMAKE_CXX_STANDARD", "14"); - cfg.build(); t!(File::create(&done_stamp)); @@ -906,7 +907,7 @@ impl Step for TestHelpers { type Output = (); fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/test/auxiliary/rust_test_helpers.c") + run.path("tests/auxiliary/rust_test_helpers.c") } fn make_run(run: RunConfig<'_>) { @@ -928,7 +929,7 @@ impl Step for TestHelpers { self.target }; let dst = builder.test_helpers_out(target); - let src = builder.src.join("src/test/auxiliary/rust_test_helpers.c"); + let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); if up_to_date(&src, &dst.join("librust_test_helpers.a")) { return; } @@ -957,7 +958,7 @@ impl Step for TestHelpers { .opt_level(0) .warnings(false) .debug(false) - .file(builder.src.join("src/test/auxiliary/rust_test_helpers.c")) + .file(builder.src.join("tests/auxiliary/rust_test_helpers.c")) .compile("rust_test_helpers"); } } @@ -991,7 +992,7 @@ impl Step for Sanitizers { return runtimes; } - let llvm_config = builder.ensure(Llvm { target: builder.config.build }); + let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: builder.config.build }); if builder.config.dry_run() { return runtimes; } @@ -1086,12 +1087,12 @@ fn supported_sanitizers( match &*target.triple { "aarch64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "aarch64-fuchsia" => common_libs("fuchsia", "aarch64", &["asan"]), + "aarch64-unknown-fuchsia" => common_libs("fuchsia", "aarch64", &["asan"]), "aarch64-unknown-linux-gnu" => { common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) } "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "x86_64-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), + "x86_64-unknown-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), "x86_64-unknown-freebsd" => common_libs("freebsd", "x86_64", &["asan", "msan", "tsan"]), "x86_64-unknown-netbsd" => { common_libs("netbsd", "x86_64", &["asan", "lsan", "msan", "tsan"]) diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs index 05de51f8c..e02808545 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/run.rs @@ -105,6 +105,7 @@ impl Step for BumpStage0 { fn run(self, builder: &Builder<'_>) -> Self::Output { let mut cmd = builder.tool_cmd(Tool::BumpStage0); + cmd.args(builder.config.cmd.args()); builder.run(&mut cmd); } } diff --git a/src/bootstrap/setup.rs b/src/bootstrap/setup.rs index c7f98a7d0..004601cb6 100644 --- a/src/bootstrap/setup.rs +++ b/src/bootstrap/setup.rs @@ -1,3 +1,4 @@ +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::Config; use crate::{t, VERSION}; use std::env::consts::EXE_SUFFIX; @@ -9,7 +10,7 @@ use std::process::Command; use std::str::FromStr; use std::{fmt, fs, io}; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] pub enum Profile { Compiler, Codegen, @@ -48,6 +49,16 @@ impl Profile { } out } + + pub fn as_str(&self) -> &'static str { + match self { + Profile::Compiler => "compiler", + Profile::Codegen => "codegen", + Profile::Library => "library", + Profile::Tools => "tools", + Profile::User => "user", + } + } } impl FromStr for Profile { @@ -69,24 +80,62 @@ impl FromStr for Profile { impl fmt::Display for Profile { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Profile::Compiler => write!(f, "compiler"), - Profile::Codegen => write!(f, "codegen"), - Profile::Library => write!(f, "library"), - Profile::User => write!(f, "user"), - Profile::Tools => write!(f, "tools"), + f.write_str(self.as_str()) + } +} + +impl Step for Profile { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(mut run: ShouldRun<'_>) -> ShouldRun<'_> { + for choice in Profile::all() { + run = run.alias(choice.as_str()); } + run + } + + fn make_run(run: RunConfig<'_>) { + if run.builder.config.dry_run() { + return; + } + + // for Profile, `run.paths` will have 1 and only 1 element + // this is because we only accept at most 1 path from user input. + // If user calls `x.py setup` without arguments, the interactive TUI + // will guide user to provide one. + let profile = if run.paths.len() > 1 { + // HACK: `builder` runs this step with all paths if no path was passed. + t!(interactive_path()) + } else { + run.paths + .first() + .unwrap() + .assert_single_path() + .path + .as_path() + .as_os_str() + .to_str() + .unwrap() + .parse() + .unwrap() + }; + + run.builder.ensure(profile); + } + + fn run(self, builder: &Builder<'_>) { + setup(&builder.build.config, self) } } -pub fn setup(config: &Config, profile: Option<Profile>) { - let profile = profile.unwrap_or_else(|| t!(interactive_path())); +pub fn setup(config: &Config, profile: Profile) { let stage_path = ["build", config.build.rustc_target_arg(), "stage1"].join(&MAIN_SEPARATOR.to_string()); if !rustup_installed() && profile != Profile::User { eprintln!("`rustup` is not installed; cannot link `stage1` toolchain"); - } else if stage_dir_exists(&stage_path[..]) { + } else if stage_dir_exists(&stage_path[..]) && !config.dry_run() { attempt_toolchain_link(&stage_path[..]); } @@ -95,7 +144,7 @@ pub fn setup(config: &Config, profile: Option<Profile>) { Profile::Tools => &[ "check", "build", - "test src/test/rustdoc*", + "test tests/rustdoc*", "test src/tools/clippy", "test src/tools/miri", "test src/tools/rustfmt", @@ -104,7 +153,9 @@ pub fn setup(config: &Config, profile: Option<Profile>) { Profile::User => &["dist", "build"], }; - t!(install_git_hook_maybe(&config)); + if !config.dry_run() { + t!(install_git_hook_maybe(&config)); + } println!(); @@ -144,6 +195,7 @@ fn setup_config_toml(path: &PathBuf, profile: Profile, config: &Config) { changelog-seen = {}\n", profile, VERSION ); + t!(fs::write(path, settings)); let include_path = profile.include_path(&config.src); @@ -299,7 +351,7 @@ pub fn interactive_path() -> io::Result<Profile> { Ok(template) } -// install a git hook to automatically run tidy --bless, if they want +// install a git hook to automatically run tidy, if they want fn install_git_hook_maybe(config: &Config) -> io::Result<()> { let git = t!(config.git().args(&["rev-parse", "--git-common-dir"]).output().map(|output| { assert!(output.status.success(), "failed to run `git`"); @@ -315,7 +367,7 @@ fn install_git_hook_maybe(config: &Config) -> io::Result<()> { println!(); println!( "Rust's CI will automatically fail if it doesn't pass `tidy`, the internal tool for ensuring code quality. -If you'd like, x.py can install a git hook for you that will automatically run `tidy --bless` before +If you'd like, x.py can install a git hook for you that will automatically run `test tidy` before pushing your code to ensure your code is up to par. If you decide later that this behavior is undesirable, simply delete the `pre-push` file from .git/hooks." ); diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 39cedfdac..6078e39ac 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -11,6 +11,7 @@ use std::iter; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use crate::builder::crate_description; use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; use crate::cache::Interned; use crate::compile; @@ -377,6 +378,7 @@ impl Step for RustAnalyzer { SourceType::InTree, &["sysroot-abi".to_owned()], ); + cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES); let dir = builder.src.join(workspace_path); // needed by rust-analyzer to find its own text fixtures, cf. @@ -689,7 +691,7 @@ impl Step for CompiletestTest { // We need `ToolStd` for the locally-built sysroot because // compiletest uses unstable features of the `test` crate. builder.ensure(compile::Std::new(compiler, host)); - let cargo = tool::prepare_tool_cargo( + let mut cargo = tool::prepare_tool_cargo( builder, compiler, Mode::ToolStd, @@ -699,6 +701,7 @@ impl Step for CompiletestTest { SourceType::InTree, &[], ); + cargo.allow_features("test"); try_run(builder, &mut cargo.into()); } @@ -840,7 +843,7 @@ impl Step for RustdocJSStd { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("src/test/rustdoc-js-std") + run.suite_path("tests/rustdoc-js-std") } fn make_run(run: RunConfig<'_>) { @@ -859,10 +862,10 @@ impl Step for RustdocJSStd { .arg("--doc-folder") .arg(builder.doc_out(self.target)) .arg("--test-folder") - .arg(builder.src.join("src/test/rustdoc-js-std")); + .arg(builder.src.join("tests/rustdoc-js-std")); for path in &builder.paths { if let Some(p) = - util::is_valid_test_suite_arg(path, "src/test/rustdoc-js-std", builder) + util::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) { if !p.ends_with(".js") { eprintln!("A non-js file was given: `{}`", path.display()); @@ -878,7 +881,7 @@ impl Step for RustdocJSStd { }); builder.run(&mut command); } else { - builder.info("No nodejs found, skipping \"src/test/rustdoc-js-std\" tests"); + builder.info("No nodejs found, skipping \"tests/rustdoc-js-std\" tests"); } } } @@ -895,7 +898,7 @@ impl Step for RustdocJSNotStd { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("src/test/rustdoc-js") + run.suite_path("tests/rustdoc-js") } fn make_run(run: RunConfig<'_>) { @@ -910,11 +913,11 @@ impl Step for RustdocJSNotStd { target: self.target, mode: "js-doc-test", suite: "rustdoc-js", - path: "src/test/rustdoc-js", + path: "tests/rustdoc-js", compare_mode: None, }); } else { - builder.info("No nodejs found, skipping \"src/test/rustdoc-js\" tests"); + builder.info("No nodejs found, skipping \"tests/rustdoc-js\" tests"); } } } @@ -975,7 +978,7 @@ impl Step for RustdocGUI { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { let builder = run.builder; - let run = run.suite_path("src/test/rustdoc-gui"); + let run = run.suite_path("tests/rustdoc-gui"); run.lazy_default_condition(Box::new(move || { builder.config.nodejs.is_some() && builder @@ -1024,7 +1027,7 @@ impl Step for RustdocGUI { // We remove existing folder to be sure there won't be artifacts remaining. builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler)); - let src_path = builder.build.src.join("src/test/rustdoc-gui/src"); + let src_path = builder.build.src.join("tests/rustdoc-gui/src"); // We generate docs for the libraries present in the rustdoc-gui's src folder. for entry in src_path.read_dir().expect("read_dir call failed") { if let Ok(entry) = entry { @@ -1048,7 +1051,7 @@ impl Step for RustdocGUI { if entry.file_name() == "link_to_definition" { cargo.env("RUSTDOCFLAGS", "-Zunstable-options --generate-link-to-definition"); } else if entry.file_name() == "scrape_examples" { - cargo.arg("-Zrustdoc-scrape-examples=examples"); + cargo.arg("-Zrustdoc-scrape-examples"); } builder.run(&mut cargo); } @@ -1063,9 +1066,9 @@ impl Step for RustdocGUI { .arg("--doc-folder") .arg(out_dir.join("doc")) .arg("--tests-folder") - .arg(builder.build.src.join("src/test/rustdoc-gui")); + .arg(builder.build.src.join("tests/rustdoc-gui")); for path in &builder.paths { - if let Some(p) = util::is_valid_test_suite_arg(path, "src/test/rustdoc-gui", builder) { + if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { if !p.ends_with(".goml") { eprintln!("A non-goml file was given: `{}`", path.display()); panic!("Cannot run rustdoc-gui tests"); @@ -1142,6 +1145,40 @@ help: to skip test's attempt to check tidiness, pass `--exclude src/tools/tidy` } } +/// Runs tidy's own tests. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct TidySelfTest; + +impl Step for TidySelfTest { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("tidyselftest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(TidySelfTest); + } + + fn run(self, builder: &Builder<'_>) { + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + "src/tools/tidy", + SourceType::InTree, + &[], + ); + try_run(builder, &mut cargo.into()); + } +} + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct ExpandYamlAnchors; @@ -1273,59 +1310,51 @@ macro_rules! test_definitions { }; } -default_test!(Ui { path: "src/test/ui", mode: "ui", suite: "ui" }); +default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" }); default_test!(RunPassValgrind { - path: "src/test/run-pass-valgrind", + path: "tests/run-pass-valgrind", mode: "run-pass-valgrind", suite: "run-pass-valgrind" }); -default_test!(MirOpt { path: "src/test/mir-opt", mode: "mir-opt", suite: "mir-opt" }); +default_test!(MirOpt { path: "tests/mir-opt", mode: "mir-opt", suite: "mir-opt" }); -default_test!(Codegen { path: "src/test/codegen", mode: "codegen", suite: "codegen" }); +default_test!(Codegen { path: "tests/codegen", mode: "codegen", suite: "codegen" }); default_test!(CodegenUnits { - path: "src/test/codegen-units", + path: "tests/codegen-units", mode: "codegen-units", suite: "codegen-units" }); -default_test!(Incremental { - path: "src/test/incremental", - mode: "incremental", - suite: "incremental" -}); +default_test!(Incremental { path: "tests/incremental", mode: "incremental", suite: "incremental" }); default_test_with_compare_mode!(Debuginfo { - path: "src/test/debuginfo", + path: "tests/debuginfo", mode: "debuginfo", suite: "debuginfo", compare_mode: "split-dwarf" }); -host_test!(UiFullDeps { path: "src/test/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); +host_test!(UiFullDeps { path: "tests/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); -host_test!(Rustdoc { path: "src/test/rustdoc", mode: "rustdoc", suite: "rustdoc" }); -host_test!(RustdocUi { path: "src/test/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" }); +host_test!(Rustdoc { path: "tests/rustdoc", mode: "rustdoc", suite: "rustdoc" }); +host_test!(RustdocUi { path: "tests/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" }); -host_test!(RustdocJson { - path: "src/test/rustdoc-json", - mode: "rustdoc-json", - suite: "rustdoc-json" -}); +host_test!(RustdocJson { path: "tests/rustdoc-json", mode: "rustdoc-json", suite: "rustdoc-json" }); -host_test!(Pretty { path: "src/test/pretty", mode: "pretty", suite: "pretty" }); +host_test!(Pretty { path: "tests/pretty", mode: "pretty", suite: "pretty" }); -default_test!(RunMake { path: "src/test/run-make", mode: "run-make", suite: "run-make" }); +default_test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make" }); host_test!(RunMakeFullDeps { - path: "src/test/run-make-fulldeps", + path: "tests/run-make-fulldeps", mode: "run-make", suite: "run-make-fulldeps" }); -default_test!(Assembly { path: "src/test/assembly", mode: "assembly", suite: "assembly" }); +default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] struct Compiletest { @@ -1437,7 +1466,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the cmd.arg("--rust-demangler-path").arg(rust_demangler); } - cmd.arg("--src-base").arg(builder.src.join("src/test").join(suite)); + cmd.arg("--src-base").arg(builder.src.join("tests").join(suite)); cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); cmd.arg("--sysroot-base").arg(builder.sysroot(compiler)); cmd.arg("--stage-id").arg(format!("stage{}-{}", compiler.stage, target)); @@ -1575,7 +1604,8 @@ note: if you're sure you want to do this, please open an issue as to why. In the let mut llvm_components_passed = false; let mut copts_passed = false; if builder.config.llvm_enabled() { - let llvm_config = builder.ensure(native::Llvm { target: builder.config.build }); + let native::LlvmResult { llvm_config, .. } = + builder.ensure(native::Llvm { target: builder.config.build }); if !builder.config.dry_run() { let llvm_version = output(Command::new(&llvm_config).arg("--version")); let llvm_components = output(Command::new(&llvm_config).arg("--components")); @@ -1900,7 +1930,7 @@ impl Step for ErrorIndex { fn make_run(run: RunConfig<'_>) { // error_index_generator depends on librustdoc. Use the compiler that // is normally used to build rustdoc for other tests (like compiletest - // tests in src/test/rustdoc) so that it shares the same artifacts. + // tests in tests/rustdoc) so that it shares the same artifacts. let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); run.builder.ensure(ErrorIndex { compiler }); } @@ -2153,8 +2183,12 @@ impl Step for Crate { } builder.info(&format!( - "{} {:?} stage{} ({} -> {})", - test_kind, self.crates, compiler.stage, &compiler.host, target + "{}{} stage{} ({} -> {})", + test_kind, + crate_description(&self.crates), + compiler.stage, + &compiler.host, + target )); let _time = util::timeit(&builder); try_run(builder, &mut cargo.into()); @@ -2193,7 +2227,7 @@ impl Step for CrateRustdoc { builder.compiler(builder.top_stage, target) } else { // Use the previous stage compiler to reuse the artifacts that are - // created when running compiletest for src/test/rustdoc. If this used + // created when running compiletest for tests/rustdoc. If this used // `compiler`, then it would cause rustdoc to be built *again*, which // isn't really necessary. builder.compiler_for(builder.top_stage, target, target) @@ -2309,7 +2343,7 @@ impl Step for CrateRustdocJsonTypes { let target = self.host; // Use the previous stage compiler to reuse the artifacts that are - // created when running compiletest for src/test/rustdoc. If this used + // created when running compiletest for tests/rustdoc. If this used // `compiler`, then it would cause rustdoc to be built *again*, which // isn't really necessary. let compiler = builder.compiler_for(builder.top_stage, target, target); diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index e0be4c432..9a2100c2f 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -29,6 +29,8 @@ struct ToolBuild { is_optional_tool: bool, source_type: SourceType, extra_features: Vec<String>, + /// Nightly-only features that are allowed (comma-separated list). + allow_features: &'static str, } impl Step for ToolBuild { @@ -59,7 +61,7 @@ impl Step for ToolBuild { _ => panic!("unexpected Mode for tool build"), } - let cargo = prepare_tool_cargo( + let mut cargo = prepare_tool_cargo( builder, compiler, self.mode, @@ -69,6 +71,9 @@ impl Step for ToolBuild { self.source_type, &self.extra_features, ); + if !self.allow_features.is_empty() { + cargo.allow_features(self.allow_features); + } builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target)); let mut duplicates = Vec::new(); @@ -292,6 +297,7 @@ macro_rules! bootstrap_tool { $name:ident, $path:expr, $tool_name:expr $(,is_external_tool = $external:expr)* $(,is_unstable_tool = $unstable:expr)* + $(,allow_features = $allow_features:expr)? ; )+) => { #[derive(Copy, PartialEq, Eq, Clone)] @@ -355,6 +361,7 @@ macro_rules! bootstrap_tool { SourceType::InTree }, extra_features: vec![], + allow_features: concat!($($allow_features)*), }).expect("expected to build -- essential tool") } } @@ -368,7 +375,7 @@ bootstrap_tool!( Tidy, "src/tools/tidy", "tidy"; Linkchecker, "src/tools/linkchecker", "linkchecker"; CargoTest, "src/tools/cargotest", "cargotest"; - Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true; + Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true, allow_features = "test"; BuildManifest, "src/tools/build-manifest", "build-manifest"; RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; RustInstaller, "src/tools/rust-installer", "rust-installer", is_external_tool = true; @@ -435,6 +442,7 @@ impl Step for ErrorIndex { is_optional_tool: false, source_type: SourceType::InTree, extra_features: Vec::new(), + allow_features: "", }) .expect("expected to build -- essential tool") } @@ -471,6 +479,7 @@ impl Step for RemoteTestServer { is_optional_tool: false, source_type: SourceType::InTree, extra_features: Vec::new(), + allow_features: "", }) .expect("expected to build -- essential tool") } @@ -622,6 +631,7 @@ impl Step for Cargo { is_optional_tool: false, source_type: SourceType::Submodule, extra_features: Vec::new(), + allow_features: "", }) .expect("expected to build -- essential tool"); @@ -637,6 +647,7 @@ impl Step for Cargo { is_optional_tool: true, source_type: SourceType::Submodule, extra_features: Vec::new(), + allow_features: "", }); }; @@ -684,6 +695,7 @@ impl Step for LldWrapper { is_optional_tool: false, source_type: SourceType::InTree, extra_features: Vec::new(), + allow_features: "", }) .expect("expected to build -- essential tool"); @@ -697,6 +709,11 @@ pub struct RustAnalyzer { pub target: TargetSelection, } +impl RustAnalyzer { + pub const ALLOW_FEATURES: &str = + "proc_macro_internals,proc_macro_diagnostic,proc_macro_span,proc_macro_span_shrink"; +} + impl Step for RustAnalyzer { type Output = Option<PathBuf>; const DEFAULT: bool = true; @@ -731,6 +748,7 @@ impl Step for RustAnalyzer { extra_features: vec!["rust-analyzer/in-rust-tree".to_owned()], is_optional_tool: false, source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, }) } } @@ -747,19 +765,9 @@ impl Step for RustAnalyzerProcMacroSrv { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - // Allow building `rust-analyzer-proc-macro-srv` both as part of the `rust-analyzer` and as a stand-alone tool. run.path("src/tools/rust-analyzer") .path("src/tools/rust-analyzer/crates/proc-macro-srv-cli") - .default_condition( - builder.config.extended - && builder.config.tools.as_ref().map_or(true, |tools| { - tools.iter().any(|tool| { - tool == "rust-analyzer" || tool == "rust-analyzer-proc-macro-srv" - }) - }), - ) } fn make_run(run: RunConfig<'_>) { @@ -779,6 +787,7 @@ impl Step for RustAnalyzerProcMacroSrv { extra_features: vec!["proc-macro-srv/sysroot-abi".to_owned()], is_optional_tool: false, source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, })?; // Copy `rust-analyzer-proc-macro-srv` to `<sysroot>/libexec/` @@ -798,6 +807,7 @@ macro_rules! tool_extended { $tool_name:expr, stable = $stable:expr $(,tool_std = $tool_std:literal)? + $(,allow_features = $allow_features:expr)? ;)+) => { $( #[derive(Debug, Clone, Hash, PartialEq, Eq)] @@ -849,6 +859,7 @@ macro_rules! tool_extended { extra_features: $sel.extra_features, is_optional_tool: true, source_type: SourceType::InTree, + allow_features: concat!($($allow_features)*), }) } } diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index 582207832..93e53d383 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -255,35 +255,6 @@ pub enum CiEnv { GitHubActions, } -impl CiEnv { - /// Obtains the current CI environment. - pub fn current() -> CiEnv { - if env::var("TF_BUILD").map_or(false, |e| e == "True") { - CiEnv::AzurePipelines - } else if env::var("GITHUB_ACTIONS").map_or(false, |e| e == "true") { - CiEnv::GitHubActions - } else { - CiEnv::None - } - } - - pub fn is_ci() -> bool { - Self::current() != CiEnv::None - } - - /// If in a CI environment, forces the command to run with colors. - pub fn force_coloring_in_ci(self, cmd: &mut Command) { - if self != CiEnv::None { - // Due to use of stamp/docker, the output stream of rustbuild is not - // a TTY in CI, so coloring is by-default turned off. - // The explicit `TERM=xterm` environment is needed for - // `--color always` to actually work. This env var was lost when - // compiling through the Makefile. Very strange. - cmd.env("TERM", "xterm").args(&["--color", "always"]); - } - } -} - pub fn forcing_clang_based_tests() -> bool { if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") { match &var.to_string_lossy().to_lowercase()[..] { @@ -441,6 +412,23 @@ pub fn output(cmd: &mut Command) -> String { String::from_utf8(output.stdout).unwrap() } +pub fn output_result(cmd: &mut Command) -> Result<String, String> { + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => return Err(format!("failed to run command: {:?}: {}", cmd, e)), + }; + if !output.status.success() { + return Err(format!( + "command did not execute successfully: {:?}\n\ + expected success, got: {}\n{}", + cmd, + output.status, + String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? + )); + } + Ok(String::from_utf8(output.stdout).map_err(|err| format!("{err:?}"))?) +} + /// Returns the last-modified time for `path`, or zero if it doesn't exist. pub fn mtime(path: &Path) -> SystemTime { fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) |