diff options
Diffstat (limited to 'src/bootstrap')
39 files changed, 1177 insertions, 1043 deletions
diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index 2b2e9e9f9..ecb58a0e9 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -85,6 +85,10 @@ dependencies = [ [[package]] name = "build_helper" version = "0.1.0" +dependencies = [ + "serde", + "serde_derive", +] [[package]] name = "cc" @@ -475,9 +479,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" dependencies = [ "memchr", ] diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 85eb543e4..74b9a23fa 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -36,7 +36,7 @@ filetime = "0.2" cc = "1.0.69" libc = "0.2" hex = "0.4" -object = { version = "0.31.1", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } +object = { version = "0.32.0", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } serde = "1.0.137" # Directly use serde_derive rather than through the derive feature of serde to allow building both # in parallel and to allow serde_json and toml to start building as soon as serde has been built. diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs index 30dfa81c6..c497cabbd 100644 --- a/src/bootstrap/bin/main.rs +++ b/src/bootstrap/bin/main.rs @@ -67,7 +67,7 @@ fn main() { `cp config.example.toml config.toml`" ); } else if let Some(suggestion) = &changelog_suggestion { - println!("{}", suggestion); + println!("{suggestion}"); } let pre_commit = config.src.join(".git").join("hooks").join("pre-commit"); @@ -80,7 +80,7 @@ fn main() { `cp config.example.toml config.toml`" ); } else if let Some(suggestion) = &changelog_suggestion { - println!("{}", suggestion); + println!("{suggestion}"); } // Give a warning if the pre-commit script is in pre-commit and not pre-push. @@ -107,13 +107,13 @@ fn check_version(config: &Config) -> Option<String> { let suggestion = if let Some(seen) = config.changelog_seen { if seen != VERSION { msg.push_str("warning: there have been changes to x.py since you last updated.\n"); - format!("update `config.toml` to use `changelog-seen = {}` instead", VERSION) + format!("update `config.toml` to use `changelog-seen = {VERSION}` instead") } else { return None; } } else { msg.push_str("warning: x.py has made several changes recently you may want to look at\n"); - format!("add `changelog-seen = {}` at the top of `config.toml`", VERSION) + format!("add `changelog-seen = {VERSION}` at the top of `config.toml`") }; msg.push_str("help: consider looking at the changes in `src/bootstrap/CHANGELOG.md`\n"); diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index e87125a49..10718aeb8 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -120,7 +120,7 @@ fn main() { // Override linker if necessary. if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") { - cmd.arg(format!("-Clinker={}", host_linker)); + cmd.arg(format!("-Clinker={host_linker}")); } if env::var_os("RUSTC_HOST_FUSE_LD_LLD").is_some() { cmd.arg("-Clink-args=-fuse-ld=lld"); @@ -206,11 +206,11 @@ fn main() { env::vars().filter(|(k, _)| k.starts_with("RUST") || k.starts_with("CARGO")); let prefix = if is_test { "[RUSTC-SHIM] rustc --test" } else { "[RUSTC-SHIM] rustc" }; let prefix = match crate_name { - Some(crate_name) => format!("{} {}", prefix, crate_name), + Some(crate_name) => format!("{prefix} {crate_name}"), None => prefix.to_string(), }; for (i, (k, v)) in rust_env_vars.enumerate() { - eprintln!("{} env[{}]: {:?}={:?}", prefix, i, k, v); + eprintln!("{prefix} env[{i}]: {k:?}={v:?}"); } eprintln!("{} working directory: {}", prefix, env::current_dir().unwrap().display()); eprintln!( @@ -220,13 +220,13 @@ fn main() { env::join_paths(&dylib_path).unwrap(), cmd, ); - eprintln!("{} sysroot: {:?}", prefix, sysroot); - eprintln!("{} libdir: {:?}", prefix, libdir); + eprintln!("{prefix} sysroot: {sysroot:?}"); + eprintln!("{prefix} libdir: {libdir:?}"); } let start = Instant::now(); let (child, status) = { - let errmsg = format!("\nFailed to run:\n{:?}\n-------------", cmd); + let errmsg = format!("\nFailed to run:\n{cmd:?}\n-------------"); let mut child = cmd.spawn().expect(&errmsg); let status = child.wait().expect(&errmsg); (child, status) @@ -259,7 +259,7 @@ fn main() { // should run on success, after this block. } if verbose > 0 { - println!("\nDid not run successfully: {}\n{:?}\n-------------", status, cmd); + println!("\nDid not run successfully: {status}\n{cmd:?}\n-------------"); } if let Some(mut on_fail) = on_fail { @@ -271,7 +271,7 @@ fn main() { match status.code() { Some(i) => std::process::exit(i), None => { - eprintln!("rustc exited with {}", status); + eprintln!("rustc exited with {status}"); std::process::exit(0xfe); } } @@ -396,21 +396,20 @@ fn format_rusage_data(_child: Child) -> Option<String> { let minflt = rusage.ru_minflt; let majflt = rusage.ru_majflt; if minflt != 0 || majflt != 0 { - init_str.push_str(&format!(" page reclaims: {} page faults: {}", minflt, majflt)); + init_str.push_str(&format!(" page reclaims: {minflt} page faults: {majflt}")); } let inblock = rusage.ru_inblock; let oublock = rusage.ru_oublock; if inblock != 0 || oublock != 0 { - init_str.push_str(&format!(" fs block inputs: {} fs block outputs: {}", inblock, oublock)); + init_str.push_str(&format!(" fs block inputs: {inblock} fs block outputs: {oublock}")); } let nvcsw = rusage.ru_nvcsw; let nivcsw = rusage.ru_nivcsw; if nvcsw != 0 || nivcsw != 0 { init_str.push_str(&format!( - " voluntary ctxt switches: {} involuntary ctxt switches: {}", - nvcsw, nivcsw + " voluntary ctxt switches: {nvcsw} involuntary ctxt switches: {nivcsw}" )); } diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs index d2b85f7a6..4ecb33498 100644 --- a/src/bootstrap/bin/rustdoc.rs +++ b/src/bootstrap/bin/rustdoc.rs @@ -62,7 +62,7 @@ fn main() { } if let Ok(no_threads) = env::var("RUSTDOC_LLD_NO_THREADS") { cmd.arg("-Clink-arg=-fuse-ld=lld"); - cmd.arg(format!("-Clink-arg=-Wl,{}", no_threads)); + cmd.arg(format!("-Clink-arg=-Wl,{no_threads}")); } // Cargo doesn't pass RUSTDOCFLAGS to proc_macros: // https://github.com/rust-lang/cargo/issues/4423 @@ -82,12 +82,12 @@ fn main() { env::join_paths(&dylib_path).unwrap(), cmd, ); - eprintln!("sysroot: {:?}", sysroot); - eprintln!("libdir: {:?}", libdir); + eprintln!("sysroot: {sysroot:?}"); + eprintln!("libdir: {libdir:?}"); } std::process::exit(match cmd.status() { Ok(s) => s.code().unwrap_or(1), - Err(e) => panic!("\n\nfailed to run {:?}: {}\n\n", cmd, e), + Err(e) => panic!("\n\nfailed to run {cmd:?}: {e}\n\n"), }) } diff --git a/src/bootstrap/bolt.rs b/src/bootstrap/bolt.rs deleted file mode 100644 index 5384181ea..000000000 --- a/src/bootstrap/bolt.rs +++ /dev/null @@ -1,60 +0,0 @@ -use std::path::Path; -use std::process::Command; - -/// Uses the `llvm-bolt` binary to instrument the artifact at the given `path` with BOLT. -/// When the instrumented artifact is executed, it will generate BOLT profiles into -/// `/tmp/prof.fdata.<pid>.fdata`. -/// Creates the instrumented artifact at `output_path`. -pub fn instrument_with_bolt(path: &Path, output_path: &Path) { - let status = Command::new("llvm-bolt") - .arg("-instrument") - .arg(&path) - // Make sure that each process will write its profiles into a separate file - .arg("--instrumentation-file-append-pid") - .arg("-o") - .arg(output_path) - .status() - .expect("Could not instrument artifact using BOLT"); - - if !status.success() { - panic!("Could not instrument {} with BOLT, exit code {:?}", path.display(), status.code()); - } -} - -/// Uses the `llvm-bolt` binary to optimize the artifact at the given `path` with BOLT, -/// using merged profiles from `profile_path`. -/// -/// The recorded profiles have to be merged using the `merge-fdata` tool from LLVM and the merged -/// profile path should be then passed to this function. -/// -/// Creates the optimized artifact at `output_path`. -pub fn optimize_with_bolt(path: &Path, profile_path: &Path, output_path: &Path) { - let status = Command::new("llvm-bolt") - .arg(&path) - .arg("-data") - .arg(&profile_path) - .arg("-o") - .arg(output_path) - // Reorder basic blocks within functions - .arg("-reorder-blocks=ext-tsp") - // Reorder functions within the binary - .arg("-reorder-functions=hfsort+") - // Split function code into hot and code regions - .arg("-split-functions") - // Split as many basic blocks as possible - .arg("-split-all-cold") - // Move jump tables to a separate section - .arg("-jump-tables=move") - // Fold functions with identical code - .arg("-icf=1") - // Update DWARF debug info in the final binary - .arg("-update-debug-sections") - // Print optimization statistics - .arg("-dyno-stats") - .status() - .expect("Could not optimize artifact using BOLT"); - - if !status.success() { - panic!("Could not optimize {} with BOLT, exit code {:?}", path.display(), status.code()); - } -} diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 149350e62..f44a05a6b 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -97,7 +97,7 @@ def _download(path, url, probably_big, verbose, exception): print("downloading {}".format(url), file=sys.stderr) try: - if probably_big or verbose: + if (probably_big or verbose) and "GITHUB_ACTIONS" not in os.environ: option = "-#" else: option = "-s" @@ -256,7 +256,7 @@ def default_build_triple(verbose): if uname is None: return 'x86_64-pc-windows-msvc' - kernel, cputype, processor = uname.decode(default_encoding).split() + kernel, cputype, processor = uname.decode(default_encoding).split(maxsplit=2) # The goal here is to come up with the same triple as LLVM would, # at least for the subset of platforms we're willing to target. @@ -332,6 +332,7 @@ def default_build_triple(verbose): 'i786': 'i686', 'loongarch64': 'loongarch64', 'm68k': 'm68k', + 'csky': 'csky', 'powerpc': 'powerpc', 'powerpc64': 'powerpc64', 'powerpc64le': 'powerpc64le', @@ -472,7 +473,9 @@ class FakeArgs: class RustBuild(object): """Provide all the methods required to build Rust""" - def __init__(self, config_toml="", args=FakeArgs()): + def __init__(self, config_toml="", args=None): + if args is None: + args = FakeArgs() self.git_version = None self.nix_deps_dir = None self._should_fix_bins_and_dylibs = None @@ -914,12 +917,7 @@ class RustBuild(object): # preserve existing RUSTFLAGS env.setdefault("RUSTFLAGS", "") - # we need to explicitly add +xgot here so that we can successfully bootstrap - # a usable stage1 compiler - # FIXME: remove this if condition on the next bootstrap bump - # cfg(bootstrap) - if self.build_triple().startswith('mips'): - env["RUSTFLAGS"] += " -Ctarget-feature=+xgot" + target_features = [] if self.get_toml("crt-static", build_section) == "true": target_features += ["+crt-static"] diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py index 3c91e403d..dc06a4c97 100644 --- a/src/bootstrap/bootstrap_test.py +++ b/src/bootstrap/bootstrap_test.py @@ -4,7 +4,6 @@ Run these with `x test bootstrap`, or `python -m unittest src/bootstrap/bootstra from __future__ import absolute_import, division, print_function import os -import doctest import unittest import tempfile import hashlib @@ -16,12 +15,15 @@ from shutil import rmtree bootstrap_dir = os.path.dirname(os.path.abspath(__file__)) # For the import below, have Python search in src/bootstrap first. sys.path.insert(0, bootstrap_dir) -import bootstrap -import configure +import bootstrap # noqa: E402 +import configure # noqa: E402 -def serialize_and_parse(configure_args, bootstrap_args=bootstrap.FakeArgs()): +def serialize_and_parse(configure_args, bootstrap_args=None): from io import StringIO + if bootstrap_args is None: + bootstrap_args = bootstrap.FakeArgs() + section_order, sections, targets = configure.parse_args(configure_args) buffer = StringIO() configure.write_config_toml(buffer, section_order, targets, sections) @@ -129,7 +131,14 @@ class GenerateAndParseConfig(unittest.TestCase): class BuildBootstrap(unittest.TestCase): """Test that we generate the appropriate arguments when building bootstrap""" - def build_args(self, configure_args=[], args=[], env={}): + def build_args(self, configure_args=None, args=None, env=None): + if configure_args is None: + configure_args = [] + if args is None: + args = [] + if env is None: + env = {} + env = env.copy() env["PATH"] = os.environ["PATH"] diff --git a/src/bootstrap/build.rs b/src/bootstrap/build.rs index cd1f41802..e0e32d313 100644 --- a/src/bootstrap/build.rs +++ b/src/bootstrap/build.rs @@ -3,5 +3,5 @@ use std::env; fn main() { let host = env::var("HOST").unwrap(); println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rustc-env=BUILD_TRIPLE={}", host); + println!("cargo:rustc-env=BUILD_TRIPLE={host}"); } diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 05b66f947..b36661928 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -13,7 +13,7 @@ use std::process::Command; use std::time::{Duration, Instant}; use crate::cache::{Cache, Interned, INTERNER}; -use crate::config::{SplitDebuginfo, TargetSelection}; +use crate::config::{DryRun, SplitDebuginfo, TargetSelection}; use crate::doc; use crate::flags::{Color, Subcommand}; use crate::install; @@ -115,6 +115,43 @@ impl RunConfig<'_> { } INTERNER.intern_list(crates) } + + /// Given an `alias` selected by the `Step` and the paths passed on the command line, + /// return a list of the crates that should be built. + /// + /// Normally, people will pass *just* `library` if they pass it. + /// But it's possible (although strange) to pass something like `library std core`. + /// Build all crates anyway, as if they hadn't passed the other args. + pub fn make_run_crates(&self, alias: Alias) -> Interned<Vec<String>> { + let has_alias = + self.paths.iter().any(|set| set.assert_single_path().path.ends_with(alias.as_str())); + if !has_alias { + return self.cargo_crates_in_set(); + } + + let crates = match alias { + Alias::Library => self.builder.in_tree_crates("sysroot", Some(self.target)), + Alias::Compiler => self.builder.in_tree_crates("rustc-main", Some(self.target)), + }; + + let crate_names = crates.into_iter().map(|krate| krate.name.to_string()).collect(); + INTERNER.intern_list(crate_names) + } +} + +#[derive(Debug, Copy, Clone)] +pub enum Alias { + Library, + Compiler, +} + +impl Alias { + fn as_str(self) -> &'static str { + match self { + Alias::Library => "library", + Alias::Compiler => "compiler", + } + } } /// A description of the crates in this set, suitable for passing to `builder.info`. @@ -280,15 +317,17 @@ impl StepDescription { } fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool { - if builder.config.exclude.iter().any(|e| pathset.has(&e, builder.kind)) { - println!("Skipping {:?} because it is excluded", pathset); + if builder.config.skip.iter().any(|e| pathset.has(&e, builder.kind)) { + if !matches!(builder.config.dry_run, DryRun::SelfCheck) { + println!("Skipping {pathset:?} because it is excluded"); + } return true; } - if !builder.config.exclude.is_empty() { + if !builder.config.skip.is_empty() && !matches!(builder.config.dry_run, DryRun::SelfCheck) { builder.verbose(&format!( "{:?} not skipped for {:?} -- not in {:?}", - pathset, self.name, builder.config.exclude + pathset, self.name, builder.config.skip )); } false @@ -354,7 +393,7 @@ impl StepDescription { eprintln!( "note: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`" ); - crate::detail_exit_macro!(1); + crate::exit!(1); } } } @@ -432,8 +471,7 @@ impl<'a> ShouldRun<'a> { // `compiler` and `library` folders respectively. assert!( self.kind == Kind::Setup || !self.builder.src.join(alias).exists(), - "use `builder.path()` for real paths: {}", - alias + "use `builder.path()` for real paths: {alias}" ); self.paths.insert(PathSet::Set( std::iter::once(TaskPath { path: alias.into(), kind: Some(self.kind) }).collect(), @@ -665,6 +703,7 @@ impl<'a> Builder<'a> { llvm::Lld, llvm::CrtBeginEnd, tool::RustdocGUITest, + tool::OptimizedDist ), Kind::Check | Kind::Clippy | Kind::Fix => describe!( check::Std, @@ -897,21 +936,6 @@ impl<'a> Builder<'a> { Self::new_internal(build, kind, paths.to_owned()) } - /// Creates a new standalone builder for use outside of the normal process - pub fn new_standalone( - build: &mut Build, - kind: Kind, - paths: Vec<PathBuf>, - stage: Option<u32>, - ) -> Builder<'_> { - // FIXME: don't mutate `build` - if let Some(stage) = stage { - build.config.stage = stage; - } - - Self::new_internal(build, kind, paths.to_owned()) - } - pub fn execute_cli(&self) { self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths); } @@ -1256,7 +1280,7 @@ impl<'a> Builder<'a> { out_dir.join(target.triple).join("doc") } } - _ => panic!("doc mode {:?} not expected", mode), + _ => panic!("doc mode {mode:?} not expected"), }; let rustdoc = self.rustdoc(compiler); self.clear_if_dirty(&my_out, &rustdoc); @@ -1333,7 +1357,7 @@ impl<'a> Builder<'a> { "error: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component" ); eprintln!("help: try `rustup component add clippy`"); - crate::detail_exit_macro!(1); + crate::exit!(1); }); if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") { rustflags.arg("--cfg=bootstrap"); @@ -1602,6 +1626,7 @@ impl<'a> Builder<'a> { // fun to pass a flag to a tool to pass a flag to pass a flag to a tool // to change a flag in a binary? if self.config.rpath_enabled(target) && util::use_host_linker(target) { + let libdir = self.sysroot_libdir_relative(compiler).to_str().unwrap(); let rpath = if target.contains("apple") { // Note that we need to take one extra step on macOS to also pass // `-Wl,-instal_name,@rpath/...` to get things to work right. To @@ -1609,15 +1634,15 @@ impl<'a> Builder<'a> { // so. Note that this is definitely a hack, and we should likely // flesh out rpath support more fully in the future. rustflags.arg("-Zosx-rpath-install-name"); - Some("-Wl,-rpath,@loader_path/../lib") + Some(format!("-Wl,-rpath,@loader_path/../{libdir}")) } else if !target.contains("windows") && !target.contains("aix") { rustflags.arg("-Clink-args=-Wl,-z,origin"); - Some("-Wl,-rpath,$ORIGIN/../lib") + Some(format!("-Wl,-rpath,$ORIGIN/../{libdir}")) } else { None }; if let Some(rpath) = rpath { - rustflags.arg(&format!("-Clink-args={}", rpath)); + rustflags.arg(&format!("-Clink-args={rpath}")); } } @@ -1631,7 +1656,7 @@ impl<'a> Builder<'a> { if let Some(target_linker) = self.linker(target) { let target = crate::envify(&target.triple); - cargo.env(&format!("CARGO_TARGET_{}_LINKER", target), target_linker); + cargo.env(&format!("CARGO_TARGET_{target}_LINKER"), target_linker); } if self.is_fuse_ld_lld(target) { rustflags.arg("-Clink-args=-fuse-ld=lld"); @@ -1867,24 +1892,24 @@ impl<'a> Builder<'a> { }; let triple_underscored = target.triple.replace("-", "_"); let cc = ccacheify(&self.cc(target)); - cargo.env(format!("CC_{}", triple_underscored), &cc); + cargo.env(format!("CC_{triple_underscored}"), &cc); let cflags = self.cflags(target, GitRepo::Rustc, CLang::C).join(" "); - cargo.env(format!("CFLAGS_{}", triple_underscored), &cflags); + cargo.env(format!("CFLAGS_{triple_underscored}"), &cflags); if let Some(ar) = self.ar(target) { let ranlib = format!("{} s", ar.display()); cargo - .env(format!("AR_{}", triple_underscored), ar) - .env(format!("RANLIB_{}", triple_underscored), ranlib); + .env(format!("AR_{triple_underscored}"), ar) + .env(format!("RANLIB_{triple_underscored}"), ranlib); } if let Ok(cxx) = self.cxx(target) { let cxx = ccacheify(&cxx); let cxxflags = self.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" "); cargo - .env(format!("CXX_{}", triple_underscored), &cxx) - .env(format!("CXXFLAGS_{}", triple_underscored), cxxflags); + .env(format!("CXX_{triple_underscored}"), &cxx) + .env(format!("CXXFLAGS_{triple_underscored}"), cxxflags); } } @@ -1997,7 +2022,7 @@ impl<'a> Builder<'a> { if let Some(limit) = limit { if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm" { - rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={}", limit)); + rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}")); } } } @@ -2005,7 +2030,7 @@ impl<'a> Builder<'a> { if matches!(mode, Mode::Std) { if let Some(mir_opt_level) = self.config.rust_validate_mir_opts { rustflags.arg("-Zvalidate-mir"); - rustflags.arg(&format!("-Zmir-opt-level={}", mir_opt_level)); + rustflags.arg(&format!("-Zmir-opt-level={mir_opt_level}")); } // Always enable inlining MIR when building the standard library. // Without this flag, MIR inlining is disabled when incremental compilation is enabled. @@ -2015,6 +2040,13 @@ impl<'a> Builder<'a> { rustflags.arg("-Zinline-mir"); } + // set rustc args passed from command line + let rustc_args = + self.config.cmd.rustc_args().iter().map(|s| s.to_string()).collect::<Vec<_>>(); + if !rustc_args.is_empty() { + cargo.env("RUSTFLAGS", &rustc_args.join(" ")); + } + Cargo { command: cargo, rustflags, rustdocflags, allow_features } } @@ -2030,9 +2062,9 @@ impl<'a> Builder<'a> { continue; } let mut out = String::new(); - out += &format!("\n\nCycle in build detected when adding {:?}\n", step); + out += &format!("\n\nCycle in build detected when adding {step:?}\n"); for el in stack.iter().rev() { - out += &format!("\t{:?}\n", el); + out += &format!("\t{el:?}\n"); } panic!("{}", out); } @@ -2059,7 +2091,7 @@ impl<'a> Builder<'a> { }; if self.config.print_step_timings && !self.config.dry_run() { - let step_string = format!("{:?}", step); + let step_string = format!("{step:?}"); let brace_index = step_string.find("{").unwrap_or(0); let type_string = type_name::<S>(); println!( @@ -2095,7 +2127,7 @@ impl<'a> Builder<'a> { let desc = StepDescription::from::<S>(kind); let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); - // Avoid running steps contained in --exclude + // Avoid running steps contained in --skip for pathset in &should_run.paths { if desc.is_excluded(self, pathset) { return None; @@ -2139,7 +2171,7 @@ impl<'a> Builder<'a> { let path = path.as_ref(); self.info(&format!("Opening doc {}", path.display())); if let Err(err) = opener::open(path) { - self.info(&format!("{}\n", err)); + self.info(&format!("{err}\n")); } } } diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index 31dcee582..43b4a34fe 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -68,13 +68,17 @@ macro_rules! std { } macro_rules! doc_std { - ($host:ident => $target:ident, stage = $stage:literal) => { + ($host:ident => $target:ident, stage = $stage:literal) => {{ + let config = configure("doc", &["A"], &["A"]); + let build = Build::new(config); + let builder = Builder::new(&build); doc::Std::new( $stage, TargetSelection::from_user(stringify!($target)), + &builder, DocumentationFormat::HTML, ) - }; + }}; } macro_rules! rustc { @@ -115,7 +119,7 @@ fn test_intersection() { #[test] fn test_exclude() { let mut config = configure("test", &["A"], &["A"]); - config.exclude = vec!["src/tools/tidy".into()]; + config.skip = vec!["src/tools/tidy".into()]; let cache = run_build(&[], config); // Ensure we have really excluded tidy @@ -133,7 +137,7 @@ fn test_exclude_kind() { // Ensure our test is valid, and `test::Rustc` would be run without the exclude. assert!(run_build(&[], config.clone()).contains::<test::CrateLibrustc>()); // Ensure tests for rustc are skipped. - config.exclude = vec![path.clone()]; + config.skip = vec![path.clone()]; assert!(!run_build(&[], config.clone()).contains::<test::CrateLibrustc>()); // Ensure builds for rustc are not skipped. assert!(run_build(&[], config).contains::<compile::Rustc>()); @@ -583,6 +587,7 @@ mod dist { run: None, only_modified: false, skip: vec![], + extra_checks: None, }; let build = Build::new(config); @@ -654,6 +659,7 @@ mod dist { pass: None, run: None, only_modified: false, + extra_checks: None, }; // Make sure rustfmt binary not being found isn't an error. config.channel = "beta".to_string(); diff --git a/src/bootstrap/cache.rs b/src/bootstrap/cache.rs index 5376c4ec9..53e4ff034 100644 --- a/src/bootstrap/cache.rs +++ b/src/bootstrap/cache.rs @@ -75,7 +75,7 @@ where { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let s: &U = &*self; - f.write_fmt(format_args!("{:?}", s)) + f.write_fmt(format_args!("{s:?}")) } } @@ -236,7 +236,7 @@ impl Cache { .or_insert_with(|| Box::new(HashMap::<S, S::Output>::new())) .downcast_mut::<HashMap<S, S::Output>>() .expect("invalid type mapped"); - assert!(!stepcache.contains_key(&step), "processing {:?} a second time", step); + assert!(!stepcache.contains_key(&step), "processing {step:?} a second time"); stepcache.insert(step, value); } diff --git a/src/bootstrap/cc_detect.rs b/src/bootstrap/cc_detect.rs index ade3bfed1..2496c2a9d 100644 --- a/src/bootstrap/cc_detect.rs +++ b/src/bootstrap/cc_detect.rs @@ -196,7 +196,7 @@ fn set_compiler( '0'..='6' => {} _ => return, } - let alternative = format!("e{}", gnu_compiler); + let alternative = format!("e{gnu_compiler}"); if Command::new(&alternative).output().is_ok() { cfg.compiler(alternative); } diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 1a0f00478..bdefc41c9 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -1,10 +1,8 @@ //! Implementation of compiling the compiler and standard library, in "check"-based modes. -use crate::builder::{crate_description, Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step}; use crate::cache::Interned; -use crate::compile::{ - add_to_sysroot, make_run_crates, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, -}; +use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo}; use crate::config::TargetSelection; use crate::tool::{prepare_tool_cargo, SourceType}; use crate::INTERNER; @@ -89,7 +87,7 @@ impl Step for Std { } fn make_run(run: RunConfig<'_>) { - let crates = make_run_crates(&run, "library"); + let crates = run.make_run_crates(Alias::Library); run.builder.ensure(Std { target: run.target, crates }); } @@ -137,10 +135,11 @@ impl Step for Std { let hostdir = builder.sysroot_libdir(compiler, compiler.host); add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); } + drop(_guard); // don't run on std twice with x.py clippy // don't check test dependencies if we haven't built libtest - if builder.kind == Kind::Clippy || !self.crates.is_empty() { + if builder.kind == Kind::Clippy || !self.crates.iter().any(|krate| krate == "test") { return; } @@ -200,10 +199,11 @@ pub struct Rustc { impl Rustc { pub fn new(target: TargetSelection, builder: &Builder<'_>) -> Self { - let mut crates = vec![]; - for krate in builder.in_tree_crates("rustc-main", None) { - crates.push(krate.name.to_string()); - } + let crates = builder + .in_tree_crates("rustc-main", Some(target)) + .into_iter() + .map(|krate| krate.name.to_string()) + .collect(); Self { target, crates: INTERNER.intern_list(crates) } } } @@ -218,7 +218,7 @@ impl Step for Rustc { } fn make_run(run: RunConfig<'_>) { - let crates = make_run_crates(&run, "compiler"); + let crates = run.make_run_crates(Alias::Compiler); run.builder.ensure(Rustc { target: run.target, crates }); } @@ -307,6 +307,12 @@ impl Step for CodegenBackend { } fn run(self, builder: &Builder<'_>) { + // FIXME: remove once https://github.com/rust-lang/rust/issues/112393 is resolved + if builder.build.config.vendor && &self.backend == "gcc" { + println!("Skipping checking of `rustc_codegen_gcc` with vendoring enabled."); + return; + } + let compiler = builder.compiler(builder.top_stage, builder.config.build); let target = self.target; let backend = self.backend; @@ -322,7 +328,7 @@ impl Step for CodegenBackend { ); cargo .arg("--manifest-path") - .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend))); + .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); rustc_cargo_env(builder, &mut cargo, target, compiler.stage); let _guard = builder.msg_check(&backend, target); @@ -525,5 +531,5 @@ fn codegen_backend_stamp( ) -> PathBuf { builder .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{}-check.stamp", backend)) + .join(format!(".librustc_codegen_{backend}-check.stamp")) } diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs index c1d867a0b..7389816b4 100644 --- a/src/bootstrap/clean.rs +++ b/src/bootstrap/clean.rs @@ -26,8 +26,15 @@ impl Step for CleanAll { } fn run(self, builder: &Builder<'_>) -> Self::Output { - let Subcommand::Clean { all, .. } = builder.config.cmd else { unreachable!("wrong subcommand?") }; - clean_default(builder.build, all) + let Subcommand::Clean { all, stage } = builder.config.cmd else { + unreachable!("wrong subcommand?") + }; + + if all && stage.is_some() { + panic!("--all and --stage can't be used at the same time for `x clean`"); + } + + clean(builder.build, all, stage) } fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -84,35 +91,70 @@ clean_crate_tree! { Std, Mode::Std, "sysroot"; } -fn clean_default(build: &Build, all: bool) { +fn clean(build: &Build, all: bool, stage: Option<u32>) { if build.config.dry_run() { return; } rm_rf("tmp".as_ref()); + // Clean the entire build directory if all { rm_rf(&build.out); - } else { - rm_rf(&build.out.join("tmp")); - rm_rf(&build.out.join("dist")); - rm_rf(&build.out.join("bootstrap")); - rm_rf(&build.out.join("rustfmt.stamp")); - - for host in &build.hosts { - let entries = match build.out.join(host.triple).read_dir() { - Ok(iter) => iter, - Err(_) => continue, - }; - - for entry in entries { - let entry = t!(entry); - if entry.file_name().to_str() == Some("llvm") { - continue; - } - let path = t!(entry.path().canonicalize()); - rm_rf(&path); + return; + } + + // Clean the target stage artifacts + if let Some(stage) = stage { + clean_specific_stage(build, stage); + return; + } + + // Follow the default behaviour + clean_default(build); +} + +fn clean_specific_stage(build: &Build, stage: u32) { + for host in &build.hosts { + let entries = match build.out.join(host.triple).read_dir() { + Ok(iter) => iter, + Err(_) => continue, + }; + + for entry in entries { + let entry = t!(entry); + let stage_prefix = format!("stage{}", stage); + + // if current entry is not related with the target stage, continue + if !entry.file_name().to_str().unwrap_or("").contains(&stage_prefix) { + continue; + } + + let path = t!(entry.path().canonicalize()); + rm_rf(&path); + } + } +} + +fn clean_default(build: &Build) { + rm_rf(&build.out.join("tmp")); + rm_rf(&build.out.join("dist")); + rm_rf(&build.out.join("bootstrap")); + rm_rf(&build.out.join("rustfmt.stamp")); + + for host in &build.hosts { + let entries = match build.out.join(host.triple).read_dir() { + Ok(iter) => iter, + Err(_) => continue, + }; + + for entry in entries { + let entry = t!(entry); + if entry.file_name().to_str() == Some("llvm") { + continue; } + let path = t!(entry.path().canonicalize()); + rm_rf(&path); } } } diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 14c3ef79a..9c68e5a78 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -23,7 +23,7 @@ use crate::builder::crate_description; use crate::builder::Cargo; use crate::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; use crate::cache::{Interned, INTERNER}; -use crate::config::{LlvmLibunwind, RustcLto, TargetSelection}; +use crate::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; use crate::dist; use crate::llvm; use crate::tool::SourceType; @@ -31,6 +31,7 @@ use crate::util::get_clang_cl_resource_dir; use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date}; use crate::LLVM_TOOLS; use crate::{CLang, Compiler, DependencyType, GitRepo, Mode}; +use filetime::FileTime; #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Std { @@ -55,17 +56,6 @@ impl Std { } } -/// Given an `alias` selected by the `Step` and the paths passed on the command line, -/// return a list of the crates that should be built. -/// -/// Normally, people will pass *just* `library` if they pass it. -/// But it's possible (although strange) to pass something like `library std core`. -/// Build all crates anyway, as if they hadn't passed the other args. -pub(crate) fn make_run_crates(run: &RunConfig<'_>, alias: &str) -> Interned<Vec<String>> { - let has_alias = run.paths.iter().any(|set| set.assert_single_path().path.ends_with(alias)); - if has_alias { Default::default() } else { run.cargo_crates_in_set() } -} - impl Step for Std { type Output = (); const DEFAULT: bool = true; @@ -80,10 +70,15 @@ impl Step for Std { } fn make_run(run: RunConfig<'_>) { + // If the paths include "library", build the entire standard library. + let has_alias = + run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); + let crates = if has_alias { Default::default() } else { run.cargo_crates_in_set() }; + run.builder.ensure(Std { compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), target: run.target, - crates: make_run_crates(&run, "library"), + crates, force_recompile: false, }); } @@ -228,13 +223,6 @@ fn copy_third_party_objects( ) -> Vec<(PathBuf, DependencyType)> { let mut target_deps = vec![]; - // FIXME: remove this in 2021 - if target == "x86_64-fortanix-unknown-sgx" { - if env::var_os("X86_FORTANIX_SGX_LIBS").is_some() { - builder.info("Warning: X86_FORTANIX_SGX_LIBS environment variable is ignored, libunwind is now compiled as part of rustbuild"); - } - } - if builder.config.sanitizers_enabled(target) && compiler.stage != 0 { // The sanitizers are only copied in stage1 or above, // to avoid creating dependency on LLVM. @@ -274,7 +262,7 @@ fn copy_self_contained_objects( // to using gcc from a glibc-targeting toolchain for linking. // To do that we have to distribute musl startup objects as a part of Rust toolchain // and link with them manually in the self-contained mode. - if target.contains("musl") { + if target.contains("musl") && !target.contains("unikraft") { let srcdir = builder.musl_libdir(target).unwrap_or_else(|| { panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple) }); @@ -300,13 +288,14 @@ fn copy_self_contained_objects( let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained); target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); } - } else if target.ends_with("-wasi") { + } else if target.contains("-wasi") { let srcdir = builder .wasi_root(target) .unwrap_or_else(|| { panic!("Target {:?} does not have a \"wasi-root\" key", target.triple) }) - .join("lib/wasm32-wasi"); + .join("lib") + .join(target.to_string().replace("-preview1", "")); for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { copy_and_stamp( builder, @@ -336,6 +325,10 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car cargo.env("MACOSX_DEPLOYMENT_TARGET", target); } + if let Some(path) = builder.config.profiler_path(target) { + cargo.env("LLVM_PROFILER_RT_LIB", path); + } + // Determine if we're going to compile in optimized C intrinsics to // the `compiler-builtins` crate. These intrinsics live in LLVM's // `compiler-rt` repository, but our `src/llvm-project` submodule isn't @@ -406,9 +399,13 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car } } - if target.ends_with("-wasi") { + if target.contains("-wasi") { if let Some(p) = builder.wasi_root(target) { - let root = format!("native={}/lib/wasm32-wasi", p.to_str().unwrap()); + let root = format!( + "native={}/lib/{}", + p.to_str().unwrap(), + target.to_string().replace("-preview1", "") + ); cargo.rustflag("-L").rustflag(&root); } } @@ -508,6 +505,49 @@ impl Step for StdLink { }; add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + + // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0` + // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta, + // and is not set to a custom path. + if compiler.stage == 0 + && builder + .build + .config + .initial_rustc + .starts_with(builder.out.join(&compiler.host.triple).join("stage0/bin")) + { + // Copy bin files from stage0/bin to stage0-sysroot/bin + let sysroot = builder.out.join(&compiler.host.triple).join("stage0-sysroot"); + + let host = compiler.host.triple; + let stage0_bin_dir = builder.out.join(&host).join("stage0/bin"); + let sysroot_bin_dir = sysroot.join("bin"); + t!(fs::create_dir_all(&sysroot_bin_dir)); + builder.cp_r(&stage0_bin_dir, &sysroot_bin_dir); + + // Copy all *.so files from stage0/lib to stage0-sysroot/lib + let stage0_lib_dir = builder.out.join(&host).join("stage0/lib"); + if let Ok(files) = fs::read_dir(&stage0_lib_dir) { + for file in files { + let file = t!(file); + let path = file.path(); + if path.is_file() && is_dylib(&file.file_name().into_string().unwrap()) { + builder.copy(&path, &sysroot.join("lib").join(path.file_name().unwrap())); + } + } + } + + // Copy codegen-backends from stage0 + let sysroot_codegen_backends = builder.sysroot_codegen_backends(compiler); + t!(fs::create_dir_all(&sysroot_codegen_backends)); + let stage0_codegen_backends = builder + .out + .join(&host) + .join("stage0/lib/rustlib") + .join(&host) + .join("codegen-backends"); + builder.cp_r(&stage0_codegen_backends, &sysroot_codegen_backends); + } } } @@ -645,8 +685,8 @@ fn cp_rustc_component_to_ci_sysroot( contents: Vec<String>, ) { let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); + let ci_rustc_dir = builder.config.ci_rustc_dir(); - let ci_rustc_dir = builder.out.join(&*builder.build.build.triple).join("ci-rustc"); for file in contents { let src = ci_rustc_dir.join(&file); let dst = sysroot.join(file); @@ -785,7 +825,7 @@ impl Step for Rustc { let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-generate={}", path)); + cargo.rustflag(&format!("-Cprofile-generate={path}")); // Apparently necessary to avoid overflowing the counters during // a Cargo build profile cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); @@ -795,7 +835,7 @@ impl Step for Rustc { } } else if let Some(path) = &builder.config.rust_profile_use { if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-use={}", path)); + cargo.rustflag(&format!("-Cprofile-use={path}")); cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); true } else { @@ -828,7 +868,7 @@ impl Step for Rustc { RustcLto::Fat => "fat", _ => unreachable!(), }; - cargo.rustflag(&format!("-Clto={}", lto_type)); + cargo.rustflag(&format!("-Clto={lto_type}")); cargo.rustflag("-Cembed-bitcode=yes"); } RustcLto::ThinLocal => { /* Do nothing, this is the default */ } @@ -853,16 +893,30 @@ impl Step for Rustc { compiler.host, target, ); + let stamp = librustc_stamp(builder, compiler, target); run_cargo( builder, cargo, vec![], - &librustc_stamp(builder, compiler, target), + &stamp, vec![], false, true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. ); + // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain + // unexpected debuginfo from dependencies, for example from the C++ standard library used in + // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with + // debuginfo (via the debuginfo level of the executables using it): strip this debuginfo + // away after the fact. + if builder.config.rust_debuginfo_level_rustc == DebuginfoLevel::None + && builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None + { + let target_root_dir = stamp.parent().unwrap(); + let rustc_driver = target_root_dir.join("librustc_driver.so"); + strip_debug(builder, target, &rustc_driver); + } + builder.ensure(RustcLink::from_rustc( self, builder.compiler(compiler.stage, builder.config.build), @@ -1079,7 +1133,7 @@ fn needs_codegen_config(run: &RunConfig<'_>) -> bool { needs_codegen_cfg } -const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; +pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) { @@ -1162,7 +1216,7 @@ impl Step for CodegenBackend { let mut cargo = builder.cargo(compiler, Mode::Codegen, SourceType::InTree, target, "build"); cargo .arg("--manifest-path") - .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend))); + .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); rustc_cargo_env(builder, &mut cargo, target, compiler.stage); let tmp_stamp = out_dir.join(".tmp.stamp"); @@ -1267,7 +1321,7 @@ fn codegen_backend_stamp( ) -> PathBuf { builder .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{}.stamp", backend)) + .join(format!(".librustc_codegen_{backend}.stamp")) } pub fn compiler_file( @@ -1282,7 +1336,7 @@ pub fn compiler_file( } let mut cmd = Command::new(compiler); cmd.args(builder.cflags(target, GitRepo::Rustc, c)); - cmd.arg(format!("-print-file-name={}", file)); + cmd.arg(format!("-print-file-name={file}")); let out = output(&mut cmd); PathBuf::from(out.trim()) } @@ -1334,6 +1388,16 @@ impl Step for Sysroot { let _ = fs::remove_dir_all(&sysroot); t!(fs::create_dir_all(&sysroot)); + // In some cases(see https://github.com/rust-lang/rust/issues/109314), when the stage0 + // compiler relies on more recent version of LLVM than the beta compiler, it may not + // be able to locate the correct LLVM in the sysroot. This situation typically occurs + // when we upgrade LLVM version while the beta compiler continues to use an older version. + // + // Make sure to add the correct version of LLVM into the stage0 sysroot. + if compiler.stage == 0 { + dist::maybe_install_llvm_target(builder, compiler.host, &sysroot); + } + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. if builder.download_rustc() && compiler.stage != 0 { assert_eq!( @@ -1381,7 +1445,7 @@ impl Step for Sysroot { // FIXME: this is wrong when compiler.host != build, but we don't support that today OsStr::new(std::env::consts::DLL_EXTENSION), ]; - let ci_rustc_dir = builder.ci_rustc_dir(builder.config.build); + let ci_rustc_dir = builder.config.ci_rustc_dir(); builder.cp_filtered(&ci_rustc_dir, &sysroot, &|path| { if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { return true; @@ -1783,7 +1847,7 @@ pub fn run_cargo( }); if !ok { - crate::detail_exit_macro!(1); + crate::exit!(1); } // Ok now we need to actually find all the files listed in `toplevel`. We've @@ -1806,10 +1870,10 @@ pub fn run_cargo( }); let path_to_add = match max { Some(triple) => triple.0.to_str().unwrap(), - None => panic!("no output generated for {:?} {:?}", prefix, extension), + None => panic!("no output generated for {prefix:?} {extension:?}"), }; if is_dylib(path_to_add) { - let candidate = format!("{}.lib", path_to_add); + let candidate = format!("{path_to_add}.lib"); let candidate = PathBuf::from(candidate); if candidate.exists() { deps.push((candidate, DependencyType::Target)); @@ -1861,10 +1925,10 @@ pub fn stream_cargo( cargo.arg(arg); } - builder.verbose(&format!("running: {:?}", cargo)); + builder.verbose(&format!("running: {cargo:?}")); let mut child = match cargo.spawn() { Ok(child) => child, - Err(e) => panic!("failed to execute command: {:?}\nerror: {}", cargo, e), + Err(e) => panic!("failed to execute command: {cargo:?}\nerror: {e}"), }; // Spawn Cargo slurping up its JSON output. We'll start building up the @@ -1877,12 +1941,12 @@ pub fn stream_cargo( Ok(msg) => { if builder.config.json_output { // Forward JSON to stdout. - println!("{}", line); + println!("{line}"); } cb(msg) } // If this was informational, just print it out and continue - Err(_) => println!("{}", line), + Err(_) => println!("{line}"), } } @@ -1890,9 +1954,8 @@ pub fn stream_cargo( let status = t!(child.wait()); if builder.is_verbose() && !status.success() { eprintln!( - "command did not execute successfully: {:?}\n\ - expected success, got: {}", - cargo, status + "command did not execute successfully: {cargo:?}\n\ + expected success, got: {status}" ); } status.success() @@ -1919,3 +1982,30 @@ pub enum CargoMessage<'a> { success: bool, }, } + +pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) { + // FIXME: to make things simpler for now, limit this to the host and target where we know + // `strip -g` is both available and will fix the issue, i.e. on a x64 linux host that is not + // cross-compiling. Expand this to other appropriate targets in the future. + if target != "x86_64-unknown-linux-gnu" || target != builder.config.build || !path.exists() { + return; + } + + let previous_mtime = FileTime::from_last_modification_time(&path.metadata().unwrap()); + // Note: `output` will propagate any errors here. + output(Command::new("strip").arg("--strip-debug").arg(path)); + + // After running `strip`, we have to set the file modification time to what it was before, + // otherwise we risk Cargo invalidating its fingerprint and rebuilding the world next time + // bootstrap is invoked. + // + // An example of this is if we run this on librustc_driver.so. In the first invocation: + // - Cargo will build librustc_driver.so (mtime of 1) + // - Cargo will build rustc-main (mtime of 2) + // - Bootstrap will strip librustc_driver.so (changing the mtime to 3). + // + // In the second invocation of bootstrap, Cargo will see that the mtime of librustc_driver.so + // is greater than the mtime of rustc-main, and will rebuild rustc-main. That will then cause + // everything else (standard library, future stages...) to be rebuilt. + t!(filetime::set_file_mtime(path, previous_mtime)); +} diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index fe932fd6b..4821d20a8 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -20,10 +20,11 @@ use std::str::FromStr; use crate::cache::{Interned, INTERNER}; use crate::cc_detect::{ndk_compiler, Language}; use crate::channel::{self, GitInfo}; +use crate::compile::CODEGEN_BACKEND_PREFIX; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags, Warnings}; use crate::util::{exe, output, t}; -use build_helper::detail_exit_macro; +use build_helper::exit; use once_cell::sync::OnceCell; use semver::Version; use serde::{Deserialize, Deserializer}; @@ -50,7 +51,7 @@ pub enum DryRun { UserSelected, } -#[derive(Copy, Clone, Default)] +#[derive(Copy, Clone, Default, PartialEq, Eq)] pub enum DebuginfoLevel { #[default] None, @@ -130,7 +131,7 @@ pub struct Config { pub sanitizers: bool, pub profiler: bool, pub omit_git_hash: bool, - pub exclude: Vec<PathBuf>, + pub skip: Vec<PathBuf>, pub include_default_paths: bool, pub rustc_error_format: Option<String>, pub json_output: bool, @@ -232,8 +233,8 @@ pub struct Config { pub llvm_profile_use: Option<String>, pub llvm_profile_generate: bool, pub llvm_libunwind_default: Option<LlvmLibunwind>, - pub llvm_bolt_profile_generate: bool, - pub llvm_bolt_profile_use: Option<String>, + + pub reproducible_artifacts: Vec<String>, pub build: TargetSelection, pub hosts: Vec<TargetSelection>, @@ -356,7 +357,7 @@ impl FromStr for LlvmLibunwind { "no" => Ok(Self::No), "in-tree" => Ok(Self::InTree), "system" => Ok(Self::System), - invalid => Err(format!("Invalid value '{}' for rust.llvm-libunwind config.", invalid)), + invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), } } } @@ -420,7 +421,7 @@ impl std::str::FromStr for RustcLto { "thin" => Ok(RustcLto::Thin), "fat" => Ok(RustcLto::Fat), "off" => Ok(RustcLto::Off), - _ => Err(format!("Invalid value for rustc LTO: {}", s)), + _ => Err(format!("Invalid value for rustc LTO: {s}")), } } } @@ -498,7 +499,7 @@ impl fmt::Display for TargetSelection { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.triple)?; if let Some(file) = self.file { - write!(f, "({})", file)?; + write!(f, "({file})")?; } Ok(()) } @@ -506,7 +507,7 @@ impl fmt::Display for TargetSelection { impl fmt::Debug for TargetSelection { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self) + write!(f, "{self}") } } @@ -533,7 +534,7 @@ pub struct Target { pub linker: Option<PathBuf>, pub ndk: Option<PathBuf>, pub sanitizers: Option<bool>, - pub profiler: Option<bool>, + pub profiler: Option<StringOrBool>, pub rpath: Option<bool>, pub crt_static: Option<bool>, pub musl_root: Option<PathBuf>, @@ -646,7 +647,7 @@ macro_rules! define_config { panic!("overriding existing option") } else { eprintln!("overriding existing option: `{}`", stringify!($field)); - detail_exit_macro!(2); + exit!(2); } } else { self.$field = other.$field; @@ -745,7 +746,7 @@ impl<T> Merge for Option<T> { panic!("overriding existing option") } else { eprintln!("overriding existing option"); - detail_exit_macro!(2); + exit!(2); } } else { *self = other; @@ -862,9 +863,9 @@ define_config! { } } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(untagged)] -enum StringOrBool { +pub enum StringOrBool { String(String), Bool(bool), } @@ -875,11 +876,16 @@ impl Default for StringOrBool { } } -#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] -#[serde(untagged)] +impl StringOrBool { + fn is_string_or_true(&self) -> bool { + matches!(self, Self::String(_) | Self::Bool(true)) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] pub enum RustOptimize { - #[serde(deserialize_with = "deserialize_and_validate_opt_level")] String(String), + Int(u8), Bool(bool), } @@ -889,26 +895,73 @@ impl Default for RustOptimize { } } -fn deserialize_and_validate_opt_level<'de, D>(d: D) -> Result<String, D::Error> -where - D: serde::de::Deserializer<'de>, -{ - let v = String::deserialize(d)?; - if ["0", "1", "2", "3", "s", "z"].iter().find(|x| **x == v).is_some() { - Ok(v) - } else { - Err(format!(r#"unrecognized option for rust optimize: "{}", expected one of "0", "1", "2", "3", "s", "z""#, v)).map_err(serde::de::Error::custom) +impl<'de> Deserialize<'de> for RustOptimize { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(OptimizeVisitor) + } +} + +struct OptimizeVisitor; + +impl<'de> serde::de::Visitor<'de> for OptimizeVisitor { + type Value = RustOptimize; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) + } + + fn visit_str<E>(self, value: &str) -> Result<Self::Value, E> + where + E: serde::de::Error, + { + if ["s", "z"].iter().find(|x| **x == value).is_some() { + Ok(RustOptimize::String(value.to_string())) + } else { + Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) + } + } + + fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E> + where + E: serde::de::Error, + { + if matches!(value, 0..=3) { + Ok(RustOptimize::Int(value as u8)) + } else { + Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) + } + } + + fn visit_bool<E>(self, value: bool) -> Result<Self::Value, E> + where + E: serde::de::Error, + { + Ok(RustOptimize::Bool(value)) } } +fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { + format!( + r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# + ) +} + impl RustOptimize { pub(crate) fn is_release(&self) -> bool { - if let RustOptimize::Bool(true) | RustOptimize::String(_) = &self { true } else { false } + match &self { + RustOptimize::Bool(true) | RustOptimize::String(_) => true, + RustOptimize::Int(i) => *i > 0, + RustOptimize::Bool(false) => false, + } } pub(crate) fn get_opt_level(&self) -> Option<String> { match &self { RustOptimize::String(s) => Some(s.clone()), + RustOptimize::Int(i) => Some(i.to_string()), RustOptimize::Bool(_) => None, } } @@ -991,7 +1044,7 @@ define_config! { llvm_libunwind: Option<String> = "llvm-libunwind", android_ndk: Option<String> = "android-ndk", sanitizers: Option<bool> = "sanitizers", - profiler: Option<bool> = "profiler", + profiler: Option<StringOrBool> = "profiler", rpath: Option<bool> = "rpath", crt_static: Option<bool> = "crt-static", musl_root: Option<String> = "musl-root", @@ -1054,7 +1107,7 @@ impl Config { .and_then(|table: toml::Value| TomlConfig::deserialize(table)) .unwrap_or_else(|err| { eprintln!("failed to parse TOML configuration '{}': {err}", file.display()); - detail_exit_macro!(2); + exit!(2); }) } Self::parse_inner(args, get_toml) @@ -1066,7 +1119,7 @@ impl Config { // Set flags. config.paths = std::mem::take(&mut flags.paths); - config.exclude = flags.exclude; + config.skip = flags.skip.into_iter().chain(flags.exclude).collect(); config.include_default_paths = flags.include_default_paths; config.rustc_error_format = flags.rustc_error_format; config.json_output = flags.json_output; @@ -1081,15 +1134,6 @@ impl Config { config.free_args = std::mem::take(&mut flags.free_args); config.llvm_profile_use = flags.llvm_profile_use; config.llvm_profile_generate = flags.llvm_profile_generate; - config.llvm_bolt_profile_generate = flags.llvm_bolt_profile_generate; - config.llvm_bolt_profile_use = flags.llvm_bolt_profile_use; - - if config.llvm_bolt_profile_generate && config.llvm_bolt_profile_use.is_some() { - eprintln!( - "Cannot use both `llvm_bolt_profile_generate` and `llvm_bolt_profile_use` at the same time" - ); - detail_exit_macro!(1); - } // Infer the rest of the configuration. @@ -1179,7 +1223,7 @@ impl Config { include_path.push("src"); include_path.push("bootstrap"); include_path.push("defaults"); - include_path.push(format!("config.{}.toml", include)); + include_path.push(format!("config.{include}.toml")); let included_toml = get_toml(&include_path); toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate); } @@ -1212,7 +1256,7 @@ impl Config { } } eprintln!("failed to parse override `{option}`: `{err}"); - detail_exit_macro!(2) + exit!(2) } toml.merge(override_toml, ReplaceOpt::Override); @@ -1328,6 +1372,25 @@ impl Config { let mut omit_git_hash = None; if let Some(rust) = toml.rust { + set(&mut config.channel, rust.channel); + + config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); + // This list is incomplete, please help by expanding it! + if config.download_rustc_commit.is_some() { + // We need the channel used by the downloaded compiler to match the one we set for rustdoc; + // otherwise rustdoc-ui tests break. + let ci_channel = t!(fs::read_to_string(config.src.join("src/ci/channel"))); + let ci_channel = ci_channel.trim_end(); + if config.channel != ci_channel + && !(config.channel == "dev" && ci_channel == "nightly") + { + panic!( + "setting rust.channel={} is incompatible with download-rustc", + config.channel + ); + } + } + debug = rust.debug; debug_assertions = rust.debug_assertions; debug_assertions_std = rust.debug_assertions_std; @@ -1339,6 +1402,7 @@ impl Config { debuginfo_level_std = rust.debuginfo_level_std; debuginfo_level_tools = rust.debuginfo_level_tools; debuginfo_level_tests = rust.debuginfo_level_tests; + config.rust_split_debuginfo = rust .split_debuginfo .as_deref() @@ -1354,7 +1418,6 @@ impl Config { set(&mut config.jemalloc, rust.jemalloc); set(&mut config.test_compare_mode, rust.test_compare_mode); set(&mut config.backtrace, rust.backtrace); - set(&mut config.channel, rust.channel); config.description = rust.description; set(&mut config.rust_dist_src, rust.dist_src); set(&mut config.verbose_tests, rust.verbose_tests); @@ -1387,16 +1450,27 @@ impl Config { .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); if let Some(ref backends) = rust.codegen_backends { - config.rust_codegen_backends = - backends.iter().map(|s| INTERNER.intern_str(s)).collect(); + let available_backends = vec!["llvm", "cranelift", "gcc"]; + + config.rust_codegen_backends = backends.iter().map(|s| { + if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { + if available_backends.contains(&backend) { + panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); + } else { + println!("help: '{s}' for 'rust.codegen-backends' might fail. \ + Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + In this case, it would be referred to as '{backend}'."); + } + } + + INTERNER.intern_str(s) + }).collect(); } config.rust_codegen_units = rust.codegen_units.map(threads_from_config); config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); - config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); - config.rust_lto = rust .lto .as_deref() @@ -1408,6 +1482,8 @@ impl Config { config.rust_profile_generate = flags.rust_profile_generate; } + config.reproducible_artifacts = flags.reproducible_artifact; + // rust_info must be set before is_ci_llvm_available() is called. let default = config.channel == "dev"; config.omit_git_hash = omit_git_hash.unwrap_or(default); @@ -1452,7 +1528,7 @@ impl Config { let asserts = llvm_assertions.unwrap_or(false); config.llvm_from_ci = match llvm.download_ci_llvm { Some(StringOrBool::String(s)) => { - assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s); + assert!(s == "if-available", "unknown option `{s}` for download-ci-llvm"); crate::llvm::is_ci_llvm_available(&config, asserts) } Some(StringOrBool::Bool(b)) => b, @@ -1508,6 +1584,11 @@ impl Config { let mut target = Target::from_triple(&triple); if let Some(ref s) = cfg.llvm_config { + if config.download_rustc_commit.is_some() && triple == &*config.build.triple { + panic!( + "setting llvm_config for the host is incompatible with download-rustc" + ); + } target.llvm_config = Some(config.src.join(s)); } target.llvm_has_rust_patches = cfg.llvm_has_rust_patches; @@ -1673,6 +1754,18 @@ impl Config { } } + /// Runs a command, printing out nice contextual information if it fails. + /// Exits if the command failed to execute at all, otherwise returns its + /// `status.success()`. + #[deprecated = "use `Builder::try_run` instead where possible"] + pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> { + if self.dry_run() { + return Ok(()); + } + self.verbose(&format!("running: {cmd:?}")); + build_helper::util::try_run(cmd, self.is_verbose()) + } + /// A git invocation which runs inside the source directory. /// /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. @@ -1709,10 +1802,10 @@ impl Config { pub(crate) fn artifact_version_part(&self, commit: &str) -> String { let (channel, version) = if self.rust_info.is_managed_git_subrepository() { let mut channel = self.git(); - channel.arg("show").arg(format!("{}:src/ci/channel", commit)); + channel.arg("show").arg(format!("{commit}:src/ci/channel")); let channel = output(&mut channel); let mut version = self.git(); - version.arg("show").arg(format!("{}:src/version", commit)); + version.arg("show").arg(format!("{commit}:src/version")); let version = output(&mut version); (channel.trim().to_owned(), version.trim().to_owned()) } else { @@ -1729,10 +1822,10 @@ impl Config { "help: consider using a git checkout or ensure these files are readable" ); if let Err(channel) = channel { - eprintln!("reading {}/src/ci/channel failed: {:?}", src, channel); + eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); } if let Err(version) = version { - eprintln!("reading {}/src/version failed: {:?}", src, version); + eprintln!("reading {src}/src/version failed: {version:?}"); } panic!(); } @@ -1778,6 +1871,12 @@ impl Config { self.out.join(&*self.build.triple).join("ci-llvm") } + /// Directory where the extracted `rustc-dev` component is stored. + pub(crate) fn ci_rustc_dir(&self) -> PathBuf { + assert!(self.download_rustc()); + self.out.join(self.build.triple).join("ci-rustc") + } + /// Determine whether llvm should be linked dynamically. /// /// If `false`, llvm should be linked statically. @@ -1813,11 +1912,11 @@ impl Config { self.download_rustc_commit().is_some() } - pub(crate) fn download_rustc_commit(&self) -> Option<&'static str> { + pub(crate) fn download_rustc_commit(&self) -> Option<&str> { static DOWNLOAD_RUSTC: OnceCell<Option<String>> = OnceCell::new(); if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() { // avoid trying to actually download the commit - return None; + return self.download_rustc_commit.as_deref(); } DOWNLOAD_RUSTC @@ -1852,7 +1951,7 @@ impl Config { pub fn verbose(&self, msg: &str) { if self.verbose > 0 { - println!("{}", msg); + println!("{msg}"); } } @@ -1864,12 +1963,24 @@ impl Config { self.target_config.values().any(|t| t.sanitizers == Some(true)) || self.sanitizers } + pub fn profiler_path(&self, target: TargetSelection) -> Option<&str> { + match self.target_config.get(&target)?.profiler.as_ref()? { + StringOrBool::String(s) => Some(s), + StringOrBool::Bool(_) => None, + } + } + pub fn profiler_enabled(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).map(|t| t.profiler).flatten().unwrap_or(self.profiler) + self.target_config + .get(&target) + .and_then(|t| t.profiler.as_ref()) + .map(StringOrBool::is_string_or_true) + .unwrap_or(self.profiler) } pub fn any_profiler_enabled(&self) -> bool { - self.target_config.values().any(|t| t.profiler == Some(true)) || self.profiler + self.target_config.values().any(|t| matches!(&t.profiler, Some(p) if p.is_string_or_true())) + || self.profiler } pub fn rpath_enabled(&self, target: TargetSelection) -> bool { @@ -1930,10 +2041,9 @@ impl Config { { let prev_version = format!("{}.{}.x", source_version.major, source_version.minor - 1); eprintln!( - "Unexpected rustc version: {}, we should use {}/{} to build source with {}", - rustc_version, prev_version, source_version, source_version + "Unexpected rustc version: {rustc_version}, we should use {prev_version}/{source_version} to build source with {source_version}" ); - detail_exit_macro!(1); + exit!(1); } } @@ -1945,7 +2055,7 @@ impl Config { Some(StringOrBool::Bool(true)) => false, Some(StringOrBool::String(s)) if s == "if-unchanged" => true, Some(StringOrBool::String(other)) => { - panic!("unrecognized option for download-rustc: {}", other) + panic!("unrecognized option for download-rustc: {other}") } }; @@ -1969,7 +2079,7 @@ impl Config { println!("help: maybe your repository history is too shallow?"); println!("help: consider disabling `download-rustc`"); println!("help: or fetch enough history to include one upstream commit"); - crate::detail_exit_macro!(1); + crate::exit!(1); } // Warn if there were changes to the compiler or standard library since the ancestor commit. diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/config/tests.rs index 732df54cd..c340bb298 100644 --- a/src/bootstrap/config/tests.rs +++ b/src/bootstrap/config/tests.rs @@ -184,7 +184,10 @@ fn rust_optimize() { assert_eq!(parse("").rust_optimize.is_release(), true); assert_eq!(parse("rust.optimize = false").rust_optimize.is_release(), false); assert_eq!(parse("rust.optimize = true").rust_optimize.is_release(), true); - assert_eq!(parse("rust.optimize = \"1\"").rust_optimize.get_opt_level(), Some("1".to_string())); + assert_eq!(parse("rust.optimize = 0").rust_optimize.is_release(), false); + assert_eq!(parse("rust.optimize = 1").rust_optimize.is_release(), true); + assert_eq!(parse("rust.optimize = 1").rust_optimize.get_opt_level(), Some("1".to_string())); + assert_eq!(parse("rust.optimize = \"s\"").rust_optimize.is_release(), true); assert_eq!(parse("rust.optimize = \"s\"").rust_optimize.get_opt_level(), Some("s".to_string())); } diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index e8eebdfb5..15e8c1eb9 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -180,7 +180,7 @@ def p(msg): def err(msg): - print("configure: error: " + msg) + print("\nconfigure: ERROR: " + msg + "\n") sys.exit(1) def is_value_list(key): @@ -544,12 +544,21 @@ def write_config_toml(writer, section_order, targets, sections): def quit_if_file_exists(file): if os.path.isfile(file): - err("Existing '" + file + "' detected.") + msg = "Existing '{}' detected. Exiting".format(file) + + # If the output object directory isn't empty, we can get these errors + host_objdir = os.environ.get("OBJDIR_ON_HOST") + if host_objdir is not None: + msg += "\nIs objdir '{}' clean?".format(host_objdir) + + err(msg) if __name__ == "__main__": # If 'config.toml' already exists, exit the script at this point quit_if_file_exists('config.toml') + if "GITHUB_ACTIONS" in os.environ: + print("::group::Configure the build") p("processing command line") # Parse all known arguments into a configuration structure that reflects the # TOML we're going to write out @@ -572,3 +581,5 @@ if __name__ == "__main__": p("") p("run `python {}/x.py --help`".format(rust_dir)) + if "GITHUB_ACTIONS" in os.environ: + print("::endgroup::") diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index b34a4b2dc..32da4ac29 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -18,9 +18,7 @@ use std::process::Command; use object::read::archive::ArchiveFile; use object::BinaryFormat; -use sha2::Digest; -use crate::bolt::{instrument_with_bolt, optimize_with_bolt}; use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; use crate::channel; @@ -106,7 +104,12 @@ impl Step for JsonDocs { /// Builds the `rust-docs-json` installer component. fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let host = self.host; - builder.ensure(crate::doc::Std::new(builder.top_stage, host, DocumentationFormat::JSON)); + builder.ensure(crate::doc::Std::new( + builder.top_stage, + host, + builder, + DocumentationFormat::JSON, + )); let dest = "share/doc/rust/json"; @@ -157,7 +160,7 @@ fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> { if let Some(file_path) = file_path { found.push(file_path); } else { - panic!("Could not find '{}' in {:?}", file, path); + panic!("Could not find '{file}' in {path:?}"); } } @@ -897,7 +900,9 @@ impl Step for Src { /// Creates the `rust-src` installer component fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - builder.update_submodule(&Path::new("src/llvm-project")); + if !builder.config.dry_run() { + builder.update_submodule(&Path::new("src/llvm-project")); + } let tarball = Tarball::new_targetless(builder, "rust-src"); @@ -1078,13 +1083,6 @@ impl Step for Cargo { tarball.add_dir(etc.join("man"), "share/man/man1"); tarball.add_legal_and_readme_to("share/doc/cargo"); - for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") { - let dirent = dirent.expect("read dir entry"); - if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") { - tarball.add_file(&dirent.path(), "libexec", 0o755); - } - } - Some(tarball.generate()) } } @@ -1480,8 +1478,8 @@ impl Step for Extended { rtf.push('}'); fn filter(contents: &str, marker: &str) -> String { - let start = format!("tool-{}-start", marker); - let end = format!("tool-{}-end", marker); + let start = format!("tool-{marker}-start"); + let end = format!("tool-{marker}-end"); let mut lines = Vec::new(); let mut omitted = false; for line in contents.lines() { @@ -1862,7 +1860,7 @@ impl Step for Extended { builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644); builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644); - builder.info(&format!("building `msi` installer with {:?}", light)); + builder.info(&format!("building `msi` installer with {light:?}")); let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple); let mut cmd = Command::new(&light); cmd.arg("-nologo") @@ -1941,19 +1939,7 @@ fn install_llvm_file(builder: &Builder<'_>, source: &Path, destination: &Path) { return; } - // After LLVM is built, we modify (instrument or optimize) the libLLVM.so library file. - // This is not done in-place so that the built LLVM files are not "tainted" with BOLT. - // We perform the instrumentation/optimization here, on the fly, just before they are being - // packaged into some destination directory. - let postprocessed = if builder.config.llvm_bolt_profile_generate { - builder.ensure(BoltInstrument::new(source.to_path_buf())) - } else if let Some(path) = &builder.config.llvm_bolt_profile_use { - builder.ensure(BoltOptimize::new(source.to_path_buf(), path.into())) - } else { - source.to_path_buf() - }; - - builder.install(&postprocessed, destination, 0o644); + builder.install(&source, destination, 0o644); } /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking. @@ -1996,7 +1982,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir { let mut cmd = Command::new(llvm_config); cmd.arg("--libfiles"); - builder.verbose(&format!("running {:?}", cmd)); + builder.verbose(&format!("running {cmd:?}")); let files = if builder.config.dry_run() { "".into() } else { output(&mut cmd) }; let build_llvm_out = &builder.llvm_out(builder.config.build); let target_llvm_out = &builder.llvm_out(target); @@ -2038,117 +2024,6 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection } } -/// Creates an output path to a BOLT-manipulated artifact for the given `file`. -/// The hash of the file is used to make sure that we don't mix BOLT artifacts amongst different -/// files with the same name. -/// -/// We need to keep the file-name the same though, to make sure that copying the manipulated file -/// to a directory will not change the final file path. -fn create_bolt_output_path(builder: &Builder<'_>, file: &Path, hash: &str) -> PathBuf { - let directory = builder.out.join("bolt").join(hash); - t!(fs::create_dir_all(&directory)); - directory.join(file.file_name().unwrap()) -} - -/// Instrument the provided file with BOLT. -/// Returns a path to the instrumented artifact. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct BoltInstrument { - file: PathBuf, - hash: String, -} - -impl BoltInstrument { - fn new(file: PathBuf) -> Self { - let mut hasher = sha2::Sha256::new(); - hasher.update(t!(fs::read(&file))); - let hash = hex::encode(hasher.finalize().as_slice()); - - Self { file, hash } - } -} - -impl Step for BoltInstrument { - type Output = PathBuf; - - const ONLY_HOSTS: bool = false; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - if builder.build.config.dry_run() { - return self.file.clone(); - } - - if builder.build.config.llvm_from_ci { - println!("warning: trying to use BOLT with LLVM from CI, this will probably not work"); - } - - println!("Instrumenting {} with BOLT", self.file.display()); - - let output_path = create_bolt_output_path(builder, &self.file, &self.hash); - if !output_path.is_file() { - instrument_with_bolt(&self.file, &output_path); - } - output_path - } -} - -/// Optimize the provided file with BOLT. -/// Returns a path to the optimized artifact. -/// -/// The hash is stored in the step to make sure that we don't optimize the same file -/// twice (even under different file paths). -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct BoltOptimize { - file: PathBuf, - profile: PathBuf, - hash: String, -} - -impl BoltOptimize { - fn new(file: PathBuf, profile: PathBuf) -> Self { - let mut hasher = sha2::Sha256::new(); - hasher.update(t!(fs::read(&file))); - hasher.update(t!(fs::read(&profile))); - let hash = hex::encode(hasher.finalize().as_slice()); - - Self { file, profile, hash } - } -} - -impl Step for BoltOptimize { - type Output = PathBuf; - - const ONLY_HOSTS: bool = false; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - if builder.build.config.dry_run() { - return self.file.clone(); - } - - if builder.build.config.llvm_from_ci { - println!("warning: trying to use BOLT with LLVM from CI, this will probably not work"); - } - - println!("Optimizing {} with BOLT", self.file.display()); - - let output_path = create_bolt_output_path(builder, &self.file, &self.hash); - if !output_path.is_file() { - optimize_with_bolt(&self.file, &self.profile, &output_path); - } - output_path - } -} - #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct LlvmTools { pub target: TargetSelection, @@ -2175,7 +2050,7 @@ impl Step for LlvmTools { /* run only if llvm-config isn't used */ if let Some(config) = builder.config.target_config.get(&target) { if let Some(ref _s) = config.llvm_config { - builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target)); + builder.info(&format!("Skipping LlvmTools ({target}): external LLVM")); return None; } } @@ -2231,7 +2106,7 @@ impl Step for RustDev { /* run only if llvm-config isn't used */ if let Some(config) = builder.config.target_config.get(&target) { if let Some(ref _s) = config.llvm_config { - builder.info(&format!("Skipping RustDev ({}): external LLVM", target)); + builder.info(&format!("Skipping RustDev ({target}): external LLVM")); return None; } } @@ -2390,8 +2265,8 @@ impl Step for ReproducibleArtifacts { tarball.add_file(path, ".", 0o644); added_anything = true; } - if let Some(path) = builder.config.llvm_bolt_profile_use.as_ref() { - tarball.add_file(path, ".", 0o644); + for profile in &builder.config.reproducible_artifacts { + tarball.add_file(profile, ".", 0o644); added_anything = true; } if added_anything { Some(tarball.generate()) } else { None } diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 5ebfe0995..505f06ed1 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -11,13 +11,12 @@ use std::fs; use std::path::{Path, PathBuf}; use crate::builder::crate_description; -use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; use crate::compile; -use crate::compile::make_run_crates; use crate::config::{Config, TargetSelection}; use crate::tool::{self, prepare_tool_cargo, SourceType, Tool}; -use crate::util::{symlink_dir, t, up_to_date}; +use crate::util::{dir_is_empty, symlink_dir, t, up_to_date}; use crate::Mode; macro_rules! submodule_helper { @@ -148,7 +147,7 @@ impl<P: Step> Step for RustbookSrc<P> { if !builder.config.dry_run() && !(up_to_date(&src, &index) || up_to_date(&rustbook, &index)) { - builder.info(&format!("Rustbook ({}) - {}", target, name)); + builder.info(&format!("Rustbook ({target}) - {name}")); let _ = fs::remove_dir_all(&out); builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(out)); @@ -198,11 +197,21 @@ impl Step for TheBook { let compiler = self.compiler; let target = self.target; + let absolute_path = builder.src.join(&relative_path); + let redirect_path = absolute_path.join("redirects"); + if !absolute_path.exists() + || !redirect_path.exists() + || dir_is_empty(&absolute_path) + || dir_is_empty(&redirect_path) + { + eprintln!("Please checkout submodule: {}", relative_path.display()); + crate::exit!(1); + } // build book builder.ensure(RustbookSrc { target, name: INTERNER.intern_str("book"), - src: INTERNER.intern_path(builder.src.join(&relative_path)), + src: INTERNER.intern_path(absolute_path.clone()), parent: Some(self), }); @@ -210,8 +219,8 @@ impl Step for TheBook { for edition in &["first-edition", "second-edition", "2018-edition"] { builder.ensure(RustbookSrc { target, - name: INTERNER.intern_string(format!("book/{}", edition)), - src: INTERNER.intern_path(builder.src.join(&relative_path).join(edition)), + name: INTERNER.intern_string(format!("book/{edition}")), + src: INTERNER.intern_path(absolute_path.join(edition)), // There should only be one book that is marked as the parent for each target, so // treat the other editions as not having a parent. parent: Option::<Self>::None, @@ -221,9 +230,12 @@ impl Step for TheBook { // build the version info page and CSS let shared_assets = builder.ensure(SharedAssets { target }); + // build the command first so we don't nest GHA groups + builder.rustdoc_cmd(compiler); + // build the redirect pages - builder.msg_doc(compiler, "book redirect pages", target); - for file in t!(fs::read_dir(builder.src.join(&relative_path).join("redirects"))) { + let _guard = builder.msg_doc(compiler, "book redirect pages", target); + for file in t!(fs::read_dir(redirect_path)) { let file = t!(file); let path = file.path(); let path = path.to_str().unwrap(); @@ -306,7 +318,7 @@ impl Step for Standalone { fn run(self, builder: &Builder<'_>) { let target = self.target; let compiler = self.compiler; - builder.msg_doc(compiler, "standalone", target); + let _guard = builder.msg_doc(compiler, "standalone", target); let out = builder.doc_out(target); t!(fs::create_dir_all(&out)); @@ -424,8 +436,18 @@ pub struct Std { } impl Std { - pub(crate) fn new(stage: u32, target: TargetSelection, format: DocumentationFormat) -> Self { - Std { stage, target, format, crates: INTERNER.intern_list(vec![]) } + pub(crate) fn new( + stage: u32, + target: TargetSelection, + builder: &Builder<'_>, + format: DocumentationFormat, + ) -> Self { + let crates = builder + .in_tree_crates("sysroot", Some(target)) + .into_iter() + .map(|krate| krate.name.to_string()) + .collect(); + Std { stage, target, format, crates: INTERNER.intern_list(crates) } } } @@ -447,7 +469,7 @@ impl Step for Std { } else { DocumentationFormat::HTML }, - crates: make_run_crates(&run, "library"), + crates: run.make_run_crates(Alias::Library), }); } @@ -455,7 +477,7 @@ impl Step for Std { /// /// This will generate all documentation for the standard library and its /// dependencies. This is largely just a wrapper around `cargo doc`. - fn run(mut self, builder: &Builder<'_>) { + fn run(self, builder: &Builder<'_>) { let stage = self.stage; let target = self.target; let out = match self.format { @@ -493,20 +515,17 @@ impl Step for Std { return; } - // Look for library/std, library/core etc in the `x.py doc` arguments and - // open the corresponding rendered docs. - if self.crates.is_empty() { - self.crates = INTERNER.intern_list(vec!["library".to_owned()]); - }; - - for requested_crate in &*self.crates { - if requested_crate == "library" { - // For `x.py doc library --open`, open `std` by default. - let index = out.join("std").join("index.html"); - builder.open_in_browser(index); - } else if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) { - let index = out.join(requested_crate).join("index.html"); - builder.open_in_browser(index); + if builder.paths.iter().any(|path| path.ends_with("library")) { + // For `x.py doc library --open`, open `std` by default. + let index = out.join("std").join("index.html"); + builder.open_in_browser(index); + } else { + for requested_crate in &*self.crates { + if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) { + let index = out.join(requested_crate).join("index.html"); + builder.open_in_browser(index); + break; + } } } } @@ -539,9 +558,6 @@ impl DocumentationFormat { } /// Build the documentation for public standard library crates. -/// -/// `requested_crates` can be used to build only a subset of the crates. If empty, all crates will -/// be built. fn doc_std( builder: &Builder<'_>, format: DocumentationFormat, @@ -554,16 +570,12 @@ fn doc_std( if builder.no_std(target) == Some(true) { panic!( "building std documentation for no_std target {target} is not supported\n\ - Set `docs = false` in the config to disable documentation, or pass `--exclude doc::library`." + Set `docs = false` in the config to disable documentation, or pass `--skip library`." ); } let compiler = builder.compiler(stage, builder.config.build); - let description = - format!("library{} in {} format", crate_description(&requested_crates), format.as_str()); - let _guard = builder.msg_doc(compiler, &description, target); - let target_doc_dir_name = if format == DocumentationFormat::JSON { "json-doc" } else { "doc" }; let target_dir = builder.stage_out(compiler, Mode::Std).join(target.triple).join(target_doc_dir_name); @@ -592,22 +604,18 @@ fn doc_std( cargo.rustdocflag("--document-private-items").rustdocflag("--document-hidden-items"); } - // HACK: because we use `--manifest-path library/sysroot/Cargo.toml`, cargo thinks we only want to document that specific crate, not its dependencies. - // Override its default. - let built_crates = if requested_crates.is_empty() { - builder - .in_tree_crates("sysroot", None) - .into_iter() - .map(|krate| krate.name.to_string()) - .collect() - } else { - requested_crates.to_vec() - }; - - for krate in built_crates { + for krate in requested_crates { + if krate == "sysroot" { + // The sysroot crate is an implementation detail, don't include it in public docs. + continue; + } cargo.arg("-p").arg(krate); } + let description = + format!("library{} in {} format", crate_description(&requested_crates), format.as_str()); + let _guard = builder.msg_doc(compiler, &description, target); + builder.run(&mut cargo.into()); builder.cp_r(&out_dir, &out); } @@ -621,20 +629,10 @@ pub struct Rustc { impl Rustc { pub(crate) fn new(stage: u32, target: TargetSelection, builder: &Builder<'_>) -> Self { - // Find dependencies for top level crates. - let root_crates = vec![ - INTERNER.intern_str("rustc_driver"), - INTERNER.intern_str("rustc_codegen_llvm"), - INTERNER.intern_str("rustc_codegen_ssa"), - ]; - let crates: Vec<_> = root_crates - .iter() - .flat_map(|krate| { - builder - .in_tree_crates(krate, Some(target)) - .into_iter() - .map(|krate| krate.name.to_string()) - }) + let crates = builder + .in_tree_crates("rustc-main", Some(target)) + .into_iter() + .map(|krate| krate.name.to_string()) .collect(); Self { stage, target, crates: INTERNER.intern_list(crates) } } @@ -656,7 +654,7 @@ impl Step for Rustc { run.builder.ensure(Rustc { stage: run.builder.top_stage, target: run.target, - crates: make_run_crates(&run, "compiler"), + crates: run.make_run_crates(Alias::Compiler), }); } @@ -666,7 +664,7 @@ impl Step for Rustc { /// Compiler documentation is distributed separately, so we make sure /// we do not merge it with the other documentation from std, test and /// proc_macros. This is largely just a wrapper around `cargo doc`. - fn run(mut self, builder: &Builder<'_>) { + fn run(self, builder: &Builder<'_>) { let stage = self.stage; let target = self.target; @@ -726,24 +724,26 @@ impl Step for Rustc { let mut to_open = None; - if self.crates.is_empty() { - self.crates = INTERNER.intern_list(vec!["rustc_driver".to_owned()]); - }; - for krate in &*self.crates { // Create all crate output directories first to make sure rustdoc uses // relative links. // FIXME: Cargo should probably do this itself. - t!(fs::create_dir_all(out_dir.join(krate))); + let dir_name = krate.replace("-", "_"); + t!(fs::create_dir_all(out_dir.join(&*dir_name))); cargo.arg("-p").arg(krate); if to_open.is_none() { - to_open = Some(krate); + to_open = Some(dir_name); } } builder.run(&mut cargo.into()); - // Let's open the first crate documentation page: - if let Some(krate) = to_open { + + if builder.paths.iter().any(|path| path.ends_with("compiler")) { + // For `x.py doc compiler --open`, open `rustc_middle` by default. + let index = out.join("rustc_middle").join("index.html"); + builder.open_in_browser(index); + } else if let Some(krate) = to_open { + // Let's open the first crate documentation page: let index = out.join(krate).join("index.html"); builder.open_in_browser(index); } @@ -812,8 +812,6 @@ macro_rules! tool_doc { SourceType::Submodule }; - builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); - // Symlink compiler docs to the output directory of rustdoc documentation. let out_dirs = [ builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"), @@ -852,6 +850,8 @@ macro_rules! tool_doc { cargo.rustdocflag("--show-type-layout"); cargo.rustdocflag("--generate-link-to-definition"); cargo.rustdocflag("-Zunstable-options"); + + let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); builder.run(&mut cargo.into()); } } @@ -894,19 +894,10 @@ tool_doc!( "-p", "cargo-credential", "-p", - "cargo-credential-1password", - "-p", "mdman", // FIXME: this trips a license check in tidy. // "-p", // "resolver-tests", - // FIXME: we should probably document these, but they're different per-platform so we can't use `tool_doc`. - // "-p", - // "cargo-credential-gnome-secret", - // "-p", - // "cargo-credential-macos-keychain", - // "-p", - // "cargo-credential-wincred", ] ); tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, ["-p", "tidy"]); @@ -975,7 +966,7 @@ impl Step for UnstableBookGen { fn run(self, builder: &Builder<'_>) { let target = self.target; - builder.info(&format!("Generating unstable book md files ({})", target)); + builder.info(&format!("Generating unstable book md files ({target})")); let out = builder.md_doc_out(target).join("unstable-book"); builder.create_dir(&out); builder.remove_dir(&out); @@ -1073,7 +1064,16 @@ impl Step for RustcBook { // config.toml), then this needs to explicitly update the dylib search // path. builder.add_rustc_lib_path(self.compiler, &mut cmd); + let doc_generator_guard = builder.msg( + Kind::Run, + self.compiler.stage, + "lint-docs", + self.compiler.host, + self.target, + ); builder.run(&mut cmd); + drop(doc_generator_guard); + // Run rustbook/mdbook to generate the HTML pages. builder.ensure(RustbookSrc { target: self.target, diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp index 120b3c9c4..ffc380579 100644 --- a/src/bootstrap/download-ci-llvm-stamp +++ b/src/bootstrap/download-ci-llvm-stamp @@ -1,4 +1,4 @@ Change this file to make users of the `download-ci-llvm` configuration download a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. -Last change is for: https://github.com/rust-lang/rust/pull/112931 +Last change is for: https://github.com/rust-lang/rust/pull/113996 diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index cb40521dd..a4135b06e 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -7,7 +7,7 @@ use std::{ process::{Command, Stdio}, }; -use build_helper::util::try_run; +use build_helper::ci::CiEnv; use once_cell::sync::OnceCell; use xz2::bufread::XzDecoder; @@ -21,6 +21,12 @@ use crate::{ static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell<bool> = OnceCell::new(); +/// `Config::try_run` wrapper for this module to avoid warnings on `try_run`, since we don't have access to a `builder` yet. +fn try_run(config: &Config, cmd: &mut Command) -> Result<(), ()> { + #[allow(deprecated)] + config.try_run(cmd) +} + /// Generic helpers that are useful anywhere in bootstrap. impl Config { pub fn is_verbose(&self) -> bool { @@ -52,24 +58,13 @@ impl Config { } /// Runs a command, printing out nice contextual information if it fails. - /// Exits if the command failed to execute at all, otherwise returns its - /// `status.success()`. - pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> { - if self.dry_run() { - return Ok(()); - } - self.verbose(&format!("running: {:?}", cmd)); - try_run(cmd, self.is_verbose()) - } - - /// Runs a command, printing out nice contextual information if it fails. /// Returns false if do not execute at all, otherwise returns its /// `status.success()`. pub(crate) fn check_run(&self, cmd: &mut Command) -> bool { if self.dry_run() { return true; } - self.verbose(&format!("running: {:?}", cmd)); + self.verbose(&format!("running: {cmd:?}")); check_run(cmd, self.is_verbose()) } @@ -156,14 +151,16 @@ impl Config { ]; } "; - nix_build_succeeded = self - .try_run(Command::new("nix-build").args(&[ + nix_build_succeeded = try_run( + self, + Command::new("nix-build").args(&[ Path::new("-E"), Path::new(NIX_EXPR), Path::new("-o"), &nix_deps_dir, - ])) - .is_ok(); + ]), + ) + .is_ok(); nix_deps_dir }); if !nix_build_succeeded { @@ -188,7 +185,7 @@ impl Config { patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); } - self.try_run(patchelf.arg(fname)).unwrap(); + let _ = try_run(self, patchelf.arg(fname)); } fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) { @@ -209,11 +206,10 @@ impl Config { } fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { - println!("downloading {}", url); + println!("downloading {url}"); // Try curl. If that fails and we are on windows, fallback to PowerShell. let mut curl = Command::new("curl"); curl.args(&[ - "-#", "-y", "30", "-Y", @@ -224,6 +220,12 @@ impl Config { "3", "-SRf", ]); + // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful. + if CiEnv::is_ci() { + curl.arg("-s"); + } else { + curl.arg("--progress-bar"); + } curl.arg(url); let f = File::create(tempfile).unwrap(); curl.stdout(Stdio::from(f)); @@ -231,7 +233,7 @@ impl Config { if self.build.contains("windows-msvc") { eprintln!("Fallback to PowerShell"); for _ in 0..3 { - if self.try_run(Command::new("PowerShell.exe").args(&[ + if try_run(self, Command::new("PowerShell.exe").args(&[ "/nologo", "-Command", "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", @@ -246,9 +248,9 @@ impl Config { } } if !help_on_error.is_empty() { - eprintln!("{}", help_on_error); + eprintln!("{help_on_error}"); } - crate::detail_exit_macro!(1); + crate::exit!(1); } } @@ -402,7 +404,11 @@ impl Config { fn ci_component_contents(&self, stamp_file: &str) -> Vec<String> { assert!(self.download_rustc()); - let ci_rustc_dir = self.out.join(&*self.build.triple).join("ci-rustc"); + if self.dry_run() { + return vec![]; + } + + let ci_rustc_dir = self.ci_rustc_dir(); let stamp_file = ci_rustc_dir.join(stamp_file); let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string()); t!(BufReader::new(contents_file).lines().collect()) @@ -419,7 +425,7 @@ impl Config { self.download_toolchain( &version, "ci-rustc", - commit, + &format!("{commit}-{}", self.llvm_assertions), &extra_components, Self::download_ci_component, ); @@ -495,8 +501,15 @@ impl Config { /// Download a single component of a CI-built toolchain (not necessarily a published nightly). // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident - fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) { - Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc") + fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) { + Self::download_component( + self, + DownloadSource::CI, + filename, + prefix, + commit_with_assertions, + "ci-rustc", + ) } fn download_component( @@ -516,11 +529,18 @@ impl Config { let bin_root = self.out.join(self.build.triple).join(destination); let tarball = cache_dir.join(&filename); let (base_url, url, should_verify) = match mode { - DownloadSource::CI => ( - self.stage0_metadata.config.artifacts_server.clone(), - format!("{key}/{filename}"), - false, - ), + DownloadSource::CI => { + let dist_server = if self.llvm_assertions { + self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone() + } else { + self.stage0_metadata.config.artifacts_server.clone() + }; + let url = format!( + "{}/{filename}", + key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap() + ); + (dist_server, url, false) + } DownloadSource::Dist => { let dist_server = env::var("RUSTUP_DIST_SERVER") .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); @@ -626,7 +646,7 @@ download-rustc = false fn download_ci_llvm(&self, llvm_sha: &str) { let llvm_assertions = self.llvm_assertions; - let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); + let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}"); let cache_dst = self.out.join("cache"); let rustc_cache = cache_dst.join(cache_prefix); if !rustc_cache.exists() { diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index a882336c3..e0291e407 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -68,7 +68,10 @@ pub struct Flags { #[arg(global(true), long, value_name = "PATH")] /// build paths to exclude - pub exclude: Vec<PathBuf>, + pub exclude: Vec<PathBuf>, // keeping for client backward compatibility + #[arg(global(true), long, value_name = "PATH")] + /// build paths to skip + pub skip: Vec<PathBuf>, #[arg(global(true), long)] /// include default paths in addition to the provided ones pub include_default_paths: bool, @@ -149,12 +152,9 @@ pub struct Flags { /// generate PGO profile with llvm built for rustc #[arg(global(true), long)] pub llvm_profile_generate: bool, - /// generate BOLT profile for LLVM build + /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. #[arg(global(true), long)] - pub llvm_bolt_profile_generate: bool, - /// use BOLT profile for LLVM build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub llvm_bolt_profile_use: Option<String>, + pub reproducible_artifact: Vec<String>, #[arg(global(true))] /// paths for the subcommand pub paths: Vec<PathBuf>, @@ -189,11 +189,11 @@ impl Flags { let build = Build::new(config); let paths = Builder::get_help(&build, subcommand); if let Some(s) = paths { - println!("{}", s); + println!("{s}"); } else { panic!("No paths available for subcommand `{}`", subcommand.as_str()); } - crate::detail_exit_macro!(0); + crate::exit!(0); } Flags::parse_from(it) @@ -321,7 +321,7 @@ pub enum Subcommand { no_fail_fast: bool, #[arg(long, value_name = "SUBSTRING")] /// skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times - skip: Vec<String>, + skip: Vec<PathBuf>, #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] /// extra arguments to be passed for the test tool being used /// (e.g. libtest, compiletest or rustdoc) @@ -339,6 +339,10 @@ pub enum Subcommand { /// whether to automatically update stderr/stdout files bless: bool, #[arg(long)] + /// comma-separated list of other files types to check (accepts py, py:lint, + /// py:fmt, shell) + extra_checks: Option<String>, + #[arg(long)] /// rerun tests even if the inputs are unchanged force_rerun: bool, #[arg(long)] @@ -366,7 +370,11 @@ pub enum Subcommand { /// Clean out build directories Clean { #[arg(long)] + /// Clean the entire build directory (not used by default) all: bool, + #[arg(long, value_name = "N")] + /// Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used. + stage: Option<u32>, }, /// Build distribution artifacts Dist, @@ -472,6 +480,13 @@ impl Subcommand { } } + pub fn extra_checks(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str), + _ => None, + } + } + pub fn only_modified(&self) -> bool { match *self { Subcommand::Test { only_modified, .. } => only_modified, @@ -538,7 +553,7 @@ pub fn get_completion<G: clap_complete::Generator>(shell: G, path: &Path) -> Opt } else { std::fs::read_to_string(path).unwrap_or_else(|_| { eprintln!("couldn't read {}", path.display()); - crate::detail_exit_macro!(1) + crate::exit!(1) }) }; let mut buf = Vec::new(); diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs index ebf068b2c..d658be0b8 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/format.rs @@ -22,7 +22,7 @@ fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl F cmd.arg("--check"); } cmd.args(paths); - let cmd_debug = format!("{:?}", cmd); + let cmd_debug = format!("{cmd:?}"); let mut cmd = cmd.spawn().expect("running rustfmt"); // poor man's async: return a closure that'll wait for rustfmt's completion move |block: bool| -> bool { @@ -40,7 +40,7 @@ fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl F code, run `./x.py fmt` instead.", cmd_debug, ); - crate::detail_exit_macro!(1); + crate::exit!(1); } true } @@ -66,13 +66,17 @@ fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, PathBuf)> { /// Return whether the format cache can be reused. fn verify_rustfmt_version(build: &Builder<'_>) -> bool { - let Some((version, stamp_file)) = get_rustfmt_version(build) else {return false;}; + let Some((version, stamp_file)) = get_rustfmt_version(build) else { + return false; + }; !program_out_of_date(&stamp_file, &version) } /// Updates the last rustfmt version used fn update_rustfmt_version(build: &Builder<'_>) { - let Some((version, stamp_file)) = get_rustfmt_version(build) else {return;}; + let Some((version, stamp_file)) = get_rustfmt_version(build) else { + return; + }; t!(std::fs::write(stamp_file, version)) } @@ -109,9 +113,9 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { } let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config)); let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config)); - let mut ignore_fmt = ignore::overrides::OverrideBuilder::new(&build.src); + let mut fmt_override = ignore::overrides::OverrideBuilder::new(&build.src); for ignore in rustfmt_config.ignore { - ignore_fmt.add(&format!("!{}", ignore)).expect(&ignore); + fmt_override.add(&format!("!{ignore}")).expect(&ignore); } let git_available = match Command::new("git") .arg("--version") @@ -148,14 +152,16 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { .map(|entry| { entry.split(' ').nth(1).expect("every git status entry should list a path") }); + let mut untracked_count = 0; for untracked_path in untracked_paths { - println!("skip untracked path {} during rustfmt invocations", untracked_path); + println!("skip untracked path {untracked_path} during rustfmt invocations"); // The leading `/` makes it an exact match against the // repository root, rather than a glob. Without that, if you // have `foo.rs` in the repository root it will also match // against anything like `compiler/rustc_foo/src/foo.rs`, // preventing the latter from being formatted. - ignore_fmt.add(&format!("!/{}", untracked_path)).expect(&untracked_path); + untracked_count += 1; + fmt_override.add(&format!("!/{untracked_path}")).expect(&untracked_path); } // Only check modified files locally to speed up runtime. // We still check all files in CI to avoid bugs in `get_modified_rs_files` letting regressions slip through; @@ -168,10 +174,25 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { println!("formatting modified file {file}"); } } else { - println!("formatting {} modified files", files.len()); + let pluralized = |count| if count > 1 { "files" } else { "file" }; + let untracked_msg = if untracked_count == 0 { + "".to_string() + } else { + format!( + ", skipped {} untracked {}", + untracked_count, + pluralized(untracked_count), + ) + }; + println!( + "formatting {} modified {}{}", + files.len(), + pluralized(files.len()), + untracked_msg + ); } for file in files { - ignore_fmt.add(&format!("/{file}")).expect(&file); + fmt_override.add(&format!("/{file}")).expect(&file); } } Ok(None) => {} @@ -192,11 +213,11 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { println!("Could not find usable git. Skipping git-aware format checks"); } - let ignore_fmt = ignore_fmt.build().unwrap(); + let fmt_override = fmt_override.build().unwrap(); let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { eprintln!("./x.py fmt is not supported on this channel"); - crate::detail_exit_macro!(1); + crate::exit!(1); }); assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); let src = build.src.clone(); @@ -248,7 +269,7 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { None => WalkBuilder::new(src.clone()), } .types(matcher) - .overrides(ignore_fmt) + .overrides(fmt_override) .build_parallel(); // there is a lot of blocking involved in spawning a child process and reading files to format. diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 0a7aff622..4396bbc51 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -27,7 +27,7 @@ use std::process::{Command, Stdio}; use std::str; use build_helper::ci::{gha, CiEnv}; -use build_helper::detail_exit_macro; +use build_helper::exit; use channel::GitInfo; use config::{DryRun, Target}; use filetime::FileTime; @@ -36,10 +36,9 @@ use once_cell::sync::OnceCell; use crate::builder::Kind; use crate::config::{LlvmLibunwind, TargetSelection}; use crate::util::{ - exe, libdir, mtime, output, run, run_suppressed, symlink_dir, try_run_suppressed, + dir_is_empty, exe, libdir, mtime, output, run, run_suppressed, symlink_dir, try_run_suppressed, }; -mod bolt; mod builder; mod cache; mod cc_detect; @@ -131,9 +130,16 @@ const EXTRA_CHECK_CFGS: &[(Option<Mode>, &'static str, Option<&[&'static str]>)] (Some(Mode::Std), "freebsd13", None), (Some(Mode::Std), "backtrace_in_libstd", None), /* Extra values not defined in the built-in targets yet, but used in std */ + // #[cfg(bootstrap)] + (Some(Mode::Std), "target_vendor", Some(&["unikraft"])), (Some(Mode::Std), "target_env", Some(&["libnx"])), - // (Some(Mode::Std), "target_os", Some(&[])), - (Some(Mode::Std), "target_arch", Some(&["asmjs", "spirv", "nvptx", "xtensa"])), + (Some(Mode::Std), "target_os", Some(&["teeos"])), + // #[cfg(bootstrap)] mips32r6, mips64r6 + ( + Some(Mode::Std), + "target_arch", + Some(&["asmjs", "spirv", "nvptx", "xtensa", "mips32r6", "mips64r6", "csky"]), + ), /* Extra names used by dependencies */ // FIXME: Used by serde_json, but we should not be triggering on external dependencies. (Some(Mode::Rustc), "no_btreemap_remove_entry", None), @@ -191,7 +197,7 @@ pub enum GitRepo { /// although most functions are implemented as free functions rather than /// methods specifically on this structure itself (to make it easier to /// organize). -#[cfg_attr(not(feature = "build-metrics"), derive(Clone))] +#[derive(Clone)] pub struct Build { /// User-specified configuration from `config.toml`. config: Config, @@ -333,7 +339,6 @@ forward! { create(path: &Path, s: &str), remove(f: &Path), tempdir() -> PathBuf, - try_run(cmd: &mut Command) -> Result<(), ()>, llvm_link_shared() -> bool, download_rustc() -> bool, initial_rustfmt() -> Option<PathBuf>, @@ -478,7 +483,7 @@ impl Build { .unwrap() .trim(); if local_release.split('.').take(2).eq(version.split('.').take(2)) { - build.verbose(&format!("auto-detected local-rebuild {}", local_release)); + build.verbose(&format!("auto-detected local-rebuild {local_release}")); build.local_rebuild = true; } @@ -531,10 +536,6 @@ impl Build { /// /// `relative_path` should be relative to the root of the git repository, not an absolute path. pub(crate) fn update_submodule(&self, relative_path: &Path) { - fn dir_is_empty(dir: &Path) -> bool { - t!(std::fs::read_dir(dir)).next().is_none() - } - if !self.config.submodules(&self.rust_info()) { return; } @@ -617,7 +618,9 @@ impl Build { } // Save any local changes, but avoid running `git stash pop` if there are none (since it will exit with an error). + #[allow(deprecated)] // diff-index reports the modifications through the exit status let has_local_modifications = self + .config .try_run( Command::new("git") .args(&["diff-index", "--quiet", "HEAD"]) @@ -709,9 +712,9 @@ impl Build { if failures.len() > 0 { eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); for failure in failures.iter() { - eprintln!(" - {}\n", failure); + eprintln!(" - {failure}\n"); } - detail_exit_macro!(1); + exit!(1); } #[cfg(feature = "build-metrics")] @@ -822,11 +825,6 @@ impl Build { self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir()) } - /// Directory where the extracted `rustc-dev` component is stored. - fn ci_rustc_dir(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("ci-rustc") - } - /// Root output directory for LLVM compiled for `target` /// /// Note that if LLVM is configured externally then the directory returned @@ -960,7 +958,7 @@ impl Build { if self.config.dry_run() { return; } - self.verbose(&format!("running: {:?}", cmd)); + self.verbose(&format!("running: {cmd:?}")); run(cmd, self.is_verbose()) } @@ -969,19 +967,43 @@ impl Build { if self.config.dry_run() { return; } - self.verbose(&format!("running: {:?}", cmd)); + self.verbose(&format!("running: {cmd:?}")); run_suppressed(cmd) } /// Runs a command, printing out nice contextual information if it fails. /// Exits if the command failed to execute at all, otherwise returns its /// `status.success()`. - fn try_run_quiet(&self, cmd: &mut Command) -> bool { + fn run_quiet_delaying_failure(&self, cmd: &mut Command) -> bool { if self.config.dry_run() { return true; } - self.verbose(&format!("running: {:?}", cmd)); - try_run_suppressed(cmd) + if !self.fail_fast { + self.verbose(&format!("running: {cmd:?}")); + if !try_run_suppressed(cmd) { + let mut failures = self.delayed_failures.borrow_mut(); + failures.push(format!("{cmd:?}")); + return false; + } + } else { + self.run_quiet(cmd); + } + true + } + + /// Runs a command, printing out contextual info if it fails, and delaying errors until the build finishes. + pub(crate) fn run_delaying_failure(&self, cmd: &mut Command) -> bool { + if !self.fail_fast { + #[allow(deprecated)] // can't use Build::try_run, that's us + if self.config.try_run(cmd).is_err() { + let mut failures = self.delayed_failures.borrow_mut(); + failures.push(format!("{cmd:?}")); + return false; + } + } else { + self.run(cmd); + } + true } pub fn is_verbose_than(&self, level: usize) -> bool { @@ -991,7 +1013,7 @@ impl Build { /// Prints a message if this build is configured in more verbose mode than `level`. fn verbose_than(&self, level: usize, msg: &str) { if self.is_verbose_than(level) { - println!("{}", msg); + println!("{msg}"); } } @@ -999,11 +1021,13 @@ impl Build { match self.config.dry_run { DryRun::SelfCheck => return, DryRun::Disabled | DryRun::UserSelected => { - println!("{}", msg); + println!("{msg}"); } } } + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] fn msg_check( &self, what: impl Display, @@ -1012,6 +1036,8 @@ impl Build { self.msg(Kind::Check, self.config.stage, what, self.config.build, target) } + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] fn msg_doc( &self, compiler: Compiler, @@ -1021,6 +1047,8 @@ impl Build { self.msg(Kind::Doc, compiler.stage, what, compiler.host, target.into()) } + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] fn msg_build( &self, compiler: Compiler, @@ -1033,6 +1061,8 @@ impl Build { /// Return a `Group` guard for a [`Step`] that is built for each `--stage`. /// /// [`Step`]: crate::builder::Step + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] fn msg( &self, action: impl Into<Kind>, @@ -1059,6 +1089,8 @@ impl Build { /// Return a `Group` guard for a [`Step`] that is only built once and isn't affected by `--stage`. /// /// [`Step`]: crate::builder::Step + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] fn msg_unstaged( &self, action: impl Into<Kind>, @@ -1070,6 +1102,8 @@ impl Build { self.group(&msg) } + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] fn msg_sysroot_tool( &self, action: impl Into<Kind>, @@ -1088,6 +1122,7 @@ impl Build { self.group(&msg) } + #[track_caller] fn group(&self, msg: &str) -> Option<gha::Group> { match self.config.dry_run { DryRun::SelfCheck => None, @@ -1111,7 +1146,7 @@ impl Build { match which { GitRepo::Rustc => { let sha = self.rust_sha().unwrap_or(&self.version); - Some(format!("/rustc/{}", sha)) + Some(format!("/rustc/{sha}")) } GitRepo::Llvm => Some(String::from("/rustc/llvm")), } @@ -1164,10 +1199,10 @@ impl Build { let map = format!("{}={}", self.src.display(), map_to); let cc = self.cc(target); if cc.ends_with("clang") || cc.ends_with("gcc") { - base.push(format!("-fdebug-prefix-map={}", map)); + base.push(format!("-fdebug-prefix-map={map}")); } else if cc.ends_with("clang-cl.exe") { base.push("-Xclang".into()); - base.push(format!("-fdebug-prefix-map={}", map)); + base.push(format!("-fdebug-prefix-map={map}")); } } base @@ -1196,9 +1231,7 @@ impl Build { } match self.cxx.borrow().get(&target) { Some(p) => Ok(p.path().into()), - None => { - Err(format!("target `{}` is not configured as a host, only as a target", target)) - } + None => Err(format!("target `{target}` is not configured as a host, only as a target")), } } @@ -1241,7 +1274,7 @@ impl Build { } let no_threads = util::lld_flag_no_threads(target.contains("windows")); - options[1] = Some(format!("-Clink-arg=-Wl,{}", no_threads)); + options[1] = Some(format!("-Clink-arg=-Wl,{no_threads}")); } IntoIterator::into_iter(options).flatten() @@ -1368,11 +1401,11 @@ impl Build { if !self.config.omit_git_hash { format!("{}-beta.{}", num, self.beta_prerelease_version()) } else { - format!("{}-beta", num) + format!("{num}-beta") } } - "nightly" => format!("{}-nightly", num), - _ => format!("{}-dev", num), + "nightly" => format!("{num}-nightly"), + _ => format!("{num}-dev"), } } @@ -1420,7 +1453,7 @@ impl Build { "stable" => num.to_string(), "beta" => "beta".to_string(), "nightly" => "nightly".to_string(), - _ => format!("{}-dev", num), + _ => format!("{num}-dev"), } } @@ -1451,7 +1484,7 @@ impl Build { /// Returns the `a.b.c` version that the given package is at. fn release_num(&self, package: &str) -> String { - let toml_file_name = self.src.join(&format!("src/tools/{}/Cargo.toml", package)); + let toml_file_name = self.src.join(&format!("src/tools/{package}/Cargo.toml")); let toml = t!(fs::read_to_string(&toml_file_name)); for line in toml.lines() { if let Some(stripped) = @@ -1461,7 +1494,7 @@ impl Build { } } - panic!("failed to find version in {}'s Cargo.toml", package) + panic!("failed to find version in {package}'s Cargo.toml") } /// Returns `true` if unstable features should be enabled for the compiler @@ -1507,6 +1540,7 @@ impl Build { } } } + ret.sort_unstable_by_key(|krate| krate.name); // reproducible order needed for tests ret } @@ -1520,7 +1554,7 @@ impl Build { "Error: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", stamp.display() ); - crate::detail_exit_macro!(1); + crate::exit!(1); } let mut paths = Vec::new(); @@ -1552,7 +1586,7 @@ impl Build { if self.config.dry_run() { return; } - self.verbose_than(1, &format!("Copy {:?} to {:?}", src, dst)); + self.verbose_than(1, &format!("Copy {src:?} to {dst:?}")); if src == dst { return; } @@ -1643,7 +1677,7 @@ impl Build { return; } let dst = dstdir.join(src.file_name().unwrap()); - self.verbose_than(1, &format!("Install {:?} to {:?}", src, dst)); + self.verbose_than(1, &format!("Install {src:?} to {dst:?}")); t!(fs::create_dir_all(dstdir)); if !src.exists() { panic!("Error: File \"{}\" not found!", src.display()); @@ -1677,7 +1711,7 @@ impl Build { let iter = match fs::read_dir(dir) { Ok(v) => v, Err(_) if self.config.dry_run() => return vec![].into_iter(), - Err(err) => panic!("could not read dir {:?}: {:?}", dir, err), + Err(err) => panic!("could not read dir {dir:?}: {err:?}"), }; iter.map(|e| t!(e)).collect::<Vec<_>>().into_iter() } @@ -1712,7 +1746,7 @@ Alternatively, set `download-ci-llvm = true` in that `[llvm]` section to download LLVM rather than building it. " ); - detail_exit_macro!(1); + exit!(1); } } diff --git a/src/bootstrap/llvm.rs b/src/bootstrap/llvm.rs index 7e27960f3..c841cb340 100644 --- a/src/bootstrap/llvm.rs +++ b/src/bootstrap/llvm.rs @@ -299,7 +299,7 @@ impl Step for Llvm { let llvm_exp_targets = match builder.config.llvm_experimental_targets { Some(ref s) => s, - None => "AVR;M68k", + None => "AVR;M68k;CSKY", }; let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" }; @@ -342,12 +342,6 @@ impl Step for Llvm { if let Some(path) = builder.config.llvm_profile_use.as_ref() { cfg.define("LLVM_PROFDATA_FILE", &path); } - if builder.config.llvm_bolt_profile_generate - || builder.config.llvm_bolt_profile_use.is_some() - { - // Relocations are required for BOLT to work. - ldflags.push_all("-Wl,-q"); - } // Disable zstd to avoid a dependency on libzstd.so. cfg.define("LLVM_ENABLE_ZSTD", "OFF"); @@ -380,12 +374,12 @@ impl Step for Llvm { cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); } - if target.starts_with("riscv") + if (target.starts_with("riscv") || target.starts_with("csky")) && !target.contains("freebsd") && !target.contains("openbsd") && !target.contains("netbsd") { - // RISC-V GCC erroneously requires linking against + // RISC-V and CSKY GCC erroneously requires linking against // `libatomic` when using 1-byte and 2-byte C++ // atomics but the LLVM build system check cannot // detect this. Therefore it is set manually here. @@ -491,25 +485,50 @@ impl Step for Llvm { cfg.build(); - // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned - // libLLVM.dylib will be built. However, llvm-config will still look - // for a versioned path like libLLVM-14.dylib. Manually create a symbolic - // link to make llvm-config happy. - if builder.llvm_link_shared() && target.contains("apple-darwin") { + // Helper to find the name of LLVM's shared library on darwin and linux. + let find_llvm_lib_name = |extension| { let mut cmd = Command::new(&res.llvm_config); let version = output(cmd.arg("--version")); let major = version.split('.').next().unwrap(); - let lib_name = match llvm_version_suffix { - Some(s) => format!("libLLVM-{}{}.dylib", major, s), - None => format!("libLLVM-{}.dylib", major), + let lib_name = match &llvm_version_suffix { + Some(version_suffix) => format!("libLLVM-{major}{version_suffix}.{extension}"), + None => format!("libLLVM-{major}.{extension}"), }; + lib_name + }; + // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned + // libLLVM.dylib will be built. However, llvm-config will still look + // for a versioned path like libLLVM-14.dylib. Manually create a symbolic + // link to make llvm-config happy. + if builder.llvm_link_shared() && target.contains("apple-darwin") { + let lib_name = find_llvm_lib_name("dylib"); let lib_llvm = out_dir.join("build").join("lib").join(lib_name); if !lib_llvm.exists() { t!(builder.symlink_file("libLLVM.dylib", &lib_llvm)); } } + // When building LLVM as a shared library on linux, it can contain unexpected debuginfo: + // some can come from the C++ standard library. Unless we're explicitly requesting LLVM to + // be built with debuginfo, strip it away after the fact, to make dist artifacts smaller. + if builder.llvm_link_shared() + && builder.config.llvm_optimize + && !builder.config.llvm_release_debuginfo + { + // Find the name of the LLVM shared library that we just built. + let lib_name = find_llvm_lib_name("so"); + + // If the shared library exists in LLVM's `/build/lib/` or `/lib/` folders, strip its + // debuginfo. + crate::compile::strip_debug(builder, target, &out_dir.join("lib").join(&lib_name)); + crate::compile::strip_debug( + builder, + target, + &out_dir.join("build").join("lib").join(&lib_name), + ); + } + t!(stamp.write()); res @@ -525,11 +544,11 @@ fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) { let version = output(cmd.arg("--version")); let mut parts = version.split('.').take(2).filter_map(|s| s.parse::<u32>().ok()); if let (Some(major), Some(_minor)) = (parts.next(), parts.next()) { - if major >= 14 { + if major >= 15 { return; } } - panic!("\n\nbad LLVM version: {}, need >=14.0\n\n", version) + panic!("\n\nbad LLVM version: {version}, need >=15.0\n\n") } fn configure_cmake( @@ -559,6 +578,8 @@ fn configure_cmake( if target.contains("netbsd") { cfg.define("CMAKE_SYSTEM_NAME", "NetBSD"); + } else if target.contains("dragonfly") { + cfg.define("CMAKE_SYSTEM_NAME", "DragonFly"); } else if target.contains("freebsd") { cfg.define("CMAKE_SYSTEM_NAME", "FreeBSD"); } else if target.contains("windows") { @@ -569,7 +590,12 @@ fn configure_cmake( cfg.define("CMAKE_SYSTEM_NAME", "SunOS"); } else if target.contains("linux") { cfg.define("CMAKE_SYSTEM_NAME", "Linux"); + } else { + builder.info( + "could not determine CMAKE_SYSTEM_NAME from the target `{target}`, build may fail", + ); } + // When cross-compiling we should also set CMAKE_SYSTEM_VERSION, but in // that case like CMake we cannot easily determine system version either. // @@ -679,10 +705,10 @@ fn configure_cmake( } } if builder.config.llvm_clang_cl.is_some() { - cflags.push(&format!(" --target={}", target)); + cflags.push(&format!(" --target={target}")); } for flag in extra_compiler_flags { - cflags.push(&format!(" {}", flag)); + cflags.push(&format!(" {flag}")); } cfg.define("CMAKE_C_FLAGS", cflags); let mut cxxflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::Cxx).join(" ").into(); @@ -691,10 +717,10 @@ fn configure_cmake( cxxflags.push(s); } if builder.config.llvm_clang_cl.is_some() { - cxxflags.push(&format!(" --target={}", target)); + cxxflags.push(&format!(" --target={target}")); } for flag in extra_compiler_flags { - cxxflags.push(&format!(" {}", flag)); + cxxflags.push(&format!(" {flag}")); } cfg.define("CMAKE_CXX_FLAGS", cxxflags); if let Some(ar) = builder.ar(target) { @@ -765,7 +791,7 @@ fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmak fn get_var(var_base: &str, host: &str, target: &str) -> Option<OsString> { let kind = if host == target { "HOST" } else { "TARGET" }; let target_u = target.replace("-", "_"); - env::var_os(&format!("{}_{}", var_base, target)) + env::var_os(&format!("{var_base}_{target}")) .or_else(|| env::var_os(&format!("{}_{}", var_base, target_u))) .or_else(|| env::var_os(&format!("{}_{}", kind, var_base))) .or_else(|| env::var_os(var_base)) @@ -1056,6 +1082,9 @@ fn supported_sanitizers( "s390x-unknown-linux-musl" => { common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) } + "x86_64-unknown-linux-ohos" => { + common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } _ => Vec::new(), } } @@ -1124,8 +1153,8 @@ impl Step for CrtBeginEnd { return out_dir; } - let crtbegin_src = builder.src.join("src/llvm-project/compiler-rt/lib/crt/crtbegin.c"); - let crtend_src = builder.src.join("src/llvm-project/compiler-rt/lib/crt/crtend.c"); + let crtbegin_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtbegin.c"); + let crtend_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtend.c"); if up_to_date(&crtbegin_src, &out_dir.join("crtbegin.o")) && up_to_date(&crtend_src, &out_dir.join("crtendS.o")) { diff --git a/src/bootstrap/metrics.rs b/src/bootstrap/metrics.rs index 5990f33b9..cf8d33dfc 100644 --- a/src/bootstrap/metrics.rs +++ b/src/bootstrap/metrics.rs @@ -7,7 +7,10 @@ use crate::builder::{Builder, Step}; use crate::util::t; use crate::Build; -use serde_derive::{Deserialize, Serialize}; +use build_helper::metrics::{ + JsonInvocation, JsonInvocationSystemStats, JsonNode, JsonRoot, JsonStepSystemStats, Test, + TestOutcome, TestSuite, TestSuiteMetadata, +}; use std::cell::RefCell; use std::fs::File; use std::io::BufWriter; @@ -37,6 +40,13 @@ pub(crate) struct BuildMetrics { state: RefCell<MetricsState>, } +/// NOTE: this isn't really cloning anything, but `x suggest` doesn't need metrics so this is probably ok. +impl Clone for BuildMetrics { + fn clone(&self) -> Self { + Self::init() + } +} + impl BuildMetrics { pub(crate) fn init() -> Self { let state = RefCell::new(MetricsState { @@ -241,98 +251,7 @@ struct StepMetrics { test_suites: Vec<TestSuite>, } -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -struct JsonRoot { - #[serde(default)] // For version 0 the field was not present. - format_version: usize, - system_stats: JsonInvocationSystemStats, - invocations: Vec<JsonInvocation>, -} - -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -struct JsonInvocation { - // Unix timestamp in seconds - // - // This is necessary to easily correlate this invocation with logs or other data. - start_time: u64, - duration_including_children_sec: f64, - children: Vec<JsonNode>, -} - -#[derive(Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum JsonNode { - RustbuildStep { - #[serde(rename = "type")] - type_: String, - debug_repr: String, - - duration_excluding_children_sec: f64, - system_stats: JsonStepSystemStats, - - children: Vec<JsonNode>, - }, - TestSuite(TestSuite), -} - -#[derive(Serialize, Deserialize)] -struct TestSuite { - metadata: TestSuiteMetadata, - tests: Vec<Test>, -} - -#[derive(Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub(crate) enum TestSuiteMetadata { - CargoPackage { - crates: Vec<String>, - target: String, - host: String, - stage: u32, - }, - Compiletest { - suite: String, - mode: String, - compare_mode: Option<String>, - target: String, - host: String, - stage: u32, - }, -} - -#[derive(Serialize, Deserialize)] -pub(crate) struct Test { - name: String, - #[serde(flatten)] - outcome: TestOutcome, -} - -#[derive(Serialize, Deserialize)] -#[serde(tag = "outcome", rename_all = "snake_case")] -pub(crate) enum TestOutcome { - Passed, - Failed, - Ignored { ignore_reason: Option<String> }, -} - -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -struct JsonInvocationSystemStats { - cpu_threads_count: usize, - cpu_model: String, - - memory_total_bytes: u64, -} - -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -struct JsonStepSystemStats { - cpu_utilization_percent: f64, -} - -#[derive(Deserialize)] +#[derive(serde_derive::Deserialize)] struct OnlyFormatVersion { #[serde(default)] // For version 0 the field was not present. format_version: usize, diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 947613796..a9865e262 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -63,7 +63,7 @@ prepare: ci-msvc-py: $(Q)$(CFG_SRC_DIR)/x.py test --stage 2 tidy ci-msvc-ps1: - $(Q)$(CFG_SRC_DIR)/x.ps1 test --stage 2 --exclude tidy + $(Q)$(CFG_SRC_DIR)/x.ps1 test --stage 2 --skip tidy ci-msvc: ci-msvc-py ci-msvc-ps1 ## MingW native builders @@ -72,7 +72,7 @@ ci-msvc: ci-msvc-py ci-msvc-ps1 ci-mingw-x: $(Q)$(CFG_SRC_DIR)/x test --stage 2 tidy ci-mingw-bootstrap: - $(Q)$(BOOTSTRAP) test --stage 2 --exclude tidy + $(Q)$(BOOTSTRAP) test --stage 2 --skip tidy ci-mingw: ci-mingw-x ci-mingw-bootstrap .PHONY: dist diff --git a/src/bootstrap/render_tests.rs b/src/bootstrap/render_tests.rs index 98a468c88..6802bf451 100644 --- a/src/bootstrap/render_tests.rs +++ b/src/bootstrap/render_tests.rs @@ -30,7 +30,7 @@ pub(crate) fn try_run_tests(builder: &Builder<'_>, cmd: &mut Command, stream: bo if !run_tests(builder, cmd, stream) { if builder.fail_fast { - crate::detail_exit_macro!(1); + crate::exit!(1); } else { let mut failures = builder.delayed_failures.borrow_mut(); failures.push(format!("{cmd:?}")); @@ -141,9 +141,9 @@ impl<'a> Renderer<'a> { self.builder.metrics.record_test( &test.name, match outcome { - Outcome::Ok | Outcome::BenchOk => crate::metrics::TestOutcome::Passed, - Outcome::Failed => crate::metrics::TestOutcome::Failed, - Outcome::Ignored { reason } => crate::metrics::TestOutcome::Ignored { + Outcome::Ok | Outcome::BenchOk => build_helper::metrics::TestOutcome::Passed, + Outcome::Failed => build_helper::metrics::TestOutcome::Failed, + Outcome::Ignored { reason } => build_helper::metrics::TestOutcome::Ignored { ignore_reason: reason.map(|s| s.to_string()), }, }, diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs index c97b75927..4082f5bb9 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/run.rs @@ -24,11 +24,9 @@ impl Step for ExpandYamlAnchors { /// anchors in them, since GitHub Actions doesn't support them. fn run(self, builder: &Builder<'_>) { builder.info("Expanding YAML anchors in the GitHub Actions configuration"); - try_run( - builder, + builder.run_delaying_failure( &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src), - ) - .unwrap(); + ); } fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -40,19 +38,6 @@ impl Step for ExpandYamlAnchors { } } -fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> Result<(), ()> { - if !builder.fail_fast { - if let Err(e) = builder.try_run(cmd) { - let mut failures = builder.delayed_failures.borrow_mut(); - failures.push(format!("{:?}", cmd)); - return Err(e); - } - } else { - builder.run(cmd); - } - Ok(()) -} - #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] pub struct BuildManifest; diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 8f5ba4273..7e83b508e 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -104,7 +104,7 @@ You should install cmake, or set `download-ci-llvm = true` in the than building it. " ); - crate::detail_exit_macro!(1); + crate::exit!(1); } } @@ -188,7 +188,7 @@ than building it. // Externally configured LLVM requires FileCheck to exist let filecheck = build.llvm_filecheck(build.build); if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { - panic!("FileCheck executable {:?} does not exist", filecheck); + panic!("FileCheck executable {filecheck:?} does not exist"); } } @@ -206,7 +206,7 @@ than building it. } // Make sure musl-root is valid - if target.contains("musl") { + if target.contains("musl") && !target.contains("unikraft") { // If this is a native target (host is also musl) and no musl-root is given, // fall back to the system toolchain in /usr before giving up if build.musl_root(*target).is_none() && build.config.build == *target { diff --git a/src/bootstrap/setup.rs b/src/bootstrap/setup.rs index 34c6ccf13..30730f504 100644 --- a/src/bootstrap/setup.rs +++ b/src/bootstrap/setup.rs @@ -32,6 +32,7 @@ static SETTINGS_HASHES: &[&str] = &[ "56e7bf011c71c5d81e0bf42e84938111847a810eee69d906bba494ea90b51922", "af1b5efe196aed007577899db9dae15d6dbc923d6fa42fa0934e68617ba9bbe0", "3468fea433c25fff60be6b71e8a215a732a7b1268b6a83bf10d024344e140541", + "47d227f424bf889b0d899b9cc992d5695e1b78c406e183cd78eafefbe5488923", ]; static RUST_ANALYZER_SETTINGS: &str = include_str!("../etc/rust_analyzer_settings.json"); @@ -92,7 +93,7 @@ impl FromStr for Profile { Ok(Profile::Tools) } "none" => Ok(Profile::None), - _ => Err(format!("unknown profile: '{}'", s)), + _ => Err(format!("unknown profile: '{s}'")), } } } @@ -167,7 +168,7 @@ pub fn setup(config: &Config, profile: Profile) { println!("To get started, try one of the following commands:"); for cmd in suggestions { - println!("- `x.py {}`", cmd); + println!("- `x.py {cmd}`"); } if profile != Profile::Dist { @@ -203,14 +204,13 @@ fn setup_config_toml(path: &PathBuf, profile: Profile, config: &Config) { "note: this will use the configuration in {}", profile.include_path(&config.src).display() ); - crate::detail_exit_macro!(1); + crate::exit!(1); } let settings = format!( "# Includes one of the default files in src/bootstrap/defaults\n\ - profile = \"{}\"\n\ - changelog-seen = {}\n", - profile, VERSION + profile = \"{profile}\"\n\ + changelog-seen = {VERSION}\n" ); t!(fs::write(path, settings)); @@ -341,7 +341,7 @@ fn ensure_stage1_toolchain_placeholder_exists(stage_path: &str) -> bool { return false; }; - let pathbuf = pathbuf.join(format!("rustc{}", EXE_SUFFIX)); + let pathbuf = pathbuf.join(format!("rustc{EXE_SUFFIX}")); if pathbuf.exists() { return true; @@ -389,12 +389,12 @@ pub fn interactive_path() -> io::Result<Profile> { io::stdin().read_line(&mut input)?; if input.is_empty() { eprintln!("EOF on stdin, when expecting answer to question. Giving up."); - crate::detail_exit_macro!(1); + crate::exit!(1); } break match parse_with_abbrev(&input) { Ok(profile) => profile, Err(err) => { - eprintln!("error: {}", err); + eprintln!("error: {err}"); eprintln!("note: press Ctrl+C to exit"); continue; } diff --git a/src/bootstrap/suggest.rs b/src/bootstrap/suggest.rs index ff20ebec2..f225104bd 100644 --- a/src/bootstrap/suggest.rs +++ b/src/bootstrap/suggest.rs @@ -4,18 +4,11 @@ use std::str::FromStr; use std::path::PathBuf; -use crate::{ - builder::{Builder, Kind}, - tool::Tool, -}; +use clap::Parser; -#[cfg(feature = "build-metrics")] -pub fn suggest(builder: &Builder<'_>, run: bool) { - panic!("`x suggest` is not supported with `build-metrics`") -} +use crate::{builder::Builder, tool::Tool}; /// Suggests a list of possible `x.py` commands to run based on modified files in branch. -#[cfg(not(feature = "build-metrics"))] pub fn suggest(builder: &Builder<'_>, run: bool) { let suggestions = builder.tool_cmd(Tool::SuggestTests).output().expect("failed to run `suggest-tests` tool"); @@ -67,12 +60,13 @@ pub fn suggest(builder: &Builder<'_>, run: bool) { if run { for sug in suggestions { - let mut build = builder.build.clone(); - - let builder = - Builder::new_standalone(&mut build, Kind::parse(&sug.0).unwrap(), sug.2, sug.1); - - builder.execute_cli() + let mut build: crate::Build = builder.build.clone(); + build.config.paths = sug.2; + build.config.cmd = crate::flags::Flags::parse_from(["x.py", sug.0]).cmd; + if let Some(stage) = sug.1 { + build.config.stage = stage; + } + build.build(); } } else { println!("help: consider using the `--run` flag to automatically run suggested tests"); diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 7fa8a4d9d..95d909c57 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -309,7 +309,7 @@ impl<'a> Tarball<'a> { let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); let package_name = self.package_name(); - self.builder.info(&format!("Dist {}", package_name)); + self.builder.info(&format!("Dist {package_name}")); let _time = crate::util::timeit(self.builder); build_cli(&self, &mut cmd); @@ -344,7 +344,7 @@ impl<'a> Tarball<'a> { .unwrap_or("gz"); GeneratedTarball { - path: crate::dist::distdir(self.builder).join(format!("{}.tar.{}", package_name, ext)), + path: crate::dist::distdir(self.builder).join(format!("{package_name}.tar.{ext}")), decompressed_output, work: self.temp_dir, } diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index eed7a584b..d0d62db08 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -4,6 +4,7 @@ //! our CI. use std::env; +use std::ffi::OsStr; use std::ffi::OsString; use std::fs; use std::iter; @@ -48,32 +49,6 @@ const MIR_OPT_BLESS_TARGET_MAPPING: &[(&str, &str)] = &[ // build for, so there is no entry for "aarch64-apple-darwin" here. ]; -fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> Result<(), ()> { - if !builder.fail_fast { - if let Err(e) = builder.try_run(cmd) { - let mut failures = builder.delayed_failures.borrow_mut(); - failures.push(format!("{:?}", cmd)); - return Err(e); - } - } else { - builder.run(cmd); - } - Ok(()) -} - -fn try_run_quiet(builder: &Builder<'_>, cmd: &mut Command) -> bool { - if !builder.fail_fast { - if !builder.try_run_quiet(cmd) { - let mut failures = builder.delayed_failures.borrow_mut(); - failures.push(format!("{:?}", cmd)); - return false; - } - } else { - builder.run_quiet(cmd); - } - true -} - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct CrateBootstrap { path: Interned<PathBuf>, @@ -117,14 +92,8 @@ impl Step for CrateBootstrap { SourceType::InTree, &[], ); - builder.info(&format!( - "{} {} stage0 ({})", - builder.kind.description(), - path, - bootstrap_host, - )); let crate_name = path.rsplit_once('/').unwrap().1; - run_cargo_test(cargo, &[], &[], crate_name, compiler, bootstrap_host, builder); + run_cargo_test(cargo, &[], &[], crate_name, crate_name, compiler, bootstrap_host, builder); } } @@ -154,15 +123,16 @@ impl Step for Linkcheck { if (hosts != targets) && !hosts.is_empty() && !targets.is_empty() { panic!( "Linkcheck currently does not support builds with different hosts and targets. -You can skip linkcheck with --exclude src/tools/linkchecker" +You can skip linkcheck with --skip src/tools/linkchecker" ); } - builder.info(&format!("Linkcheck ({})", host)); + builder.info(&format!("Linkcheck ({host})")); // Test the linkchecker itself. let bootstrap_host = builder.config.build; let compiler = builder.compiler(0, bootstrap_host); + let cargo = tool::prepare_tool_cargo( builder, compiler, @@ -173,7 +143,16 @@ You can skip linkcheck with --exclude src/tools/linkchecker" SourceType::InTree, &[], ); - run_cargo_test(cargo, &[], &[], "linkchecker", compiler, bootstrap_host, builder); + run_cargo_test( + cargo, + &[], + &[], + "linkchecker", + "linkchecker self tests", + compiler, + bootstrap_host, + builder, + ); if builder.doc_tests == DocTests::No { return; @@ -182,13 +161,14 @@ You can skip linkcheck with --exclude src/tools/linkchecker" // Build all the default documentation. builder.default_doc(&[]); + // Build the linkchecker before calling `msg`, since GHA doesn't support nested groups. + let mut linkchecker = builder.tool_cmd(Tool::Linkchecker); + // Run the linkchecker. + let _guard = + builder.msg(Kind::Test, compiler.stage, "Linkcheck", bootstrap_host, bootstrap_host); let _time = util::timeit(&builder); - try_run( - builder, - builder.tool_cmd(Tool::Linkchecker).arg(builder.out.join(host.triple).join("doc")), - ) - .unwrap(); + builder.run_delaying_failure(linkchecker.arg(builder.out.join(host.triple).join("doc"))); } fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -241,8 +221,9 @@ impl Step for HtmlCheck { builder.default_doc(&[]); builder.ensure(crate::doc::Rustc::new(builder.top_stage, self.target, builder)); - try_run(builder, builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target))) - .unwrap(); + builder.run_delaying_failure( + builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target)), + ); } } @@ -281,15 +262,13 @@ impl Step for Cargotest { let _time = util::timeit(&builder); let mut cmd = builder.tool_cmd(Tool::CargoTest); - try_run( - builder, + builder.run_delaying_failure( cmd.arg(&cargo) .arg(&out_dir) .args(builder.config.test_args()) .env("RUSTC", builder.rustc(compiler)) .env("RUSTDOC", builder.rustdoc(compiler)), - ) - .unwrap(); + ); } } @@ -340,7 +319,7 @@ impl Step for Cargo { #[cfg(feature = "build-metrics")] builder.metrics.begin_test_suite( - crate::metrics::TestSuiteMetadata::CargoPackage { + build_helper::metrics::TestSuiteMetadata::CargoPackage { crates: vec!["cargo".into()], target: self.host.triple.to_string(), host: self.host.triple.to_string(), @@ -409,7 +388,7 @@ impl Step for RustAnalyzer { cargo.env("SKIP_SLOW_TESTS", "1"); cargo.add_rustc_lib_path(builder, compiler); - run_cargo_test(cargo, &[], &[], "rust-analyzer", compiler, host, builder); + run_cargo_test(cargo, &[], &[], "rust-analyzer", "rust-analyzer", compiler, host, builder); } } @@ -458,7 +437,7 @@ impl Step for Rustfmt { cargo.add_rustc_lib_path(builder, compiler); - run_cargo_test(cargo, &[], &[], "rustfmt", compiler, host, builder); + run_cargo_test(cargo, &[], &[], "rustfmt", "rustfmt", compiler, host, builder); } } @@ -506,7 +485,16 @@ impl Step for RustDemangler { cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler); cargo.add_rustc_lib_path(builder, compiler); - run_cargo_test(cargo, &[], &[], "rust-demangler", compiler, host, builder); + run_cargo_test( + cargo, + &[], + &[], + "rust-demangler", + "rust-demangler", + compiler, + host, + builder, + ); } } @@ -550,6 +538,13 @@ impl Miri { cargo.env("RUST_BACKTRACE", "1"); let mut cargo = Command::from(cargo); + let _guard = builder.msg( + Kind::Build, + compiler.stage + 1, + "miri sysroot", + compiler.host, + compiler.host, + ); builder.run(&mut cargo); // # Determine where Miri put its sysroot. @@ -563,7 +558,7 @@ impl Miri { if builder.config.dry_run() { String::new() } else { - builder.verbose(&format!("running: {:?}", cargo)); + builder.verbose(&format!("running: {cargo:?}")); let out = cargo.output().expect("We already ran `cargo miri setup` before and that worked"); assert!(out.status.success(), "`cargo miri setup` returned with non-0 exit code"); @@ -571,7 +566,7 @@ impl Miri { let stdout = String::from_utf8(out.stdout) .expect("`cargo miri setup` stdout is not valid UTF-8"); let sysroot = stdout.trim_end(); - builder.verbose(&format!("`cargo miri setup --print-sysroot` said: {:?}", sysroot)); + builder.verbose(&format!("`cargo miri setup --print-sysroot` said: {sysroot:?}")); sysroot.to_owned() } } @@ -627,16 +622,14 @@ impl Step for Miri { SourceType::InTree, &[], ); + let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "miri", host, host); + cargo.add_rustc_lib_path(builder, compiler); // miri tests need to know about the stage sysroot cargo.env("MIRI_SYSROOT", &miri_sysroot); cargo.env("MIRI_HOST_SYSROOT", sysroot); cargo.env("MIRI", &miri); - // propagate --bless - if builder.config.cmd.bless() { - cargo.env("MIRI_BLESS", "Gesundheit"); - } // Set the target. cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); @@ -654,8 +647,8 @@ impl Step for Miri { cargo.env("MIRIFLAGS", "-O -Zmir-opt-level=4 -Cdebug-assertions=yes"); // Optimizations can change backtraces cargo.env("MIRI_SKIP_UI_CHECKS", "1"); - // `MIRI_SKIP_UI_CHECKS` and `MIRI_BLESS` are incompatible - cargo.env_remove("MIRI_BLESS"); + // `MIRI_SKIP_UI_CHECKS` and `RUSTC_BLESS` are incompatible + cargo.env_remove("RUSTC_BLESS"); // Optimizations can change error locations and remove UB so don't run `fail` tests. cargo.args(&["tests/pass", "tests/panic"]); @@ -739,7 +732,16 @@ impl Step for CompiletestTest { &[], ); cargo.allow_features("test"); - run_cargo_test(cargo, &[], &[], "compiletest", compiler, host, builder); + run_cargo_test( + cargo, + &[], + &[], + "compiletest", + "compiletest self test", + compiler, + host, + builder, + ); } } @@ -790,18 +792,16 @@ impl Step for Clippy { cargo.add_rustc_lib_path(builder, compiler); let mut cargo = prepare_cargo_test(cargo, &[], &[], "clippy", compiler, host, builder); - // propagate --bless - if builder.config.cmd.bless() { - cargo.env("BLESS", "Gesundheit"); - } + let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host); - if builder.try_run(&mut cargo).is_ok() { + #[allow(deprecated)] // Clippy reports errors if it blessed the outputs + if builder.config.try_run(&mut cargo).is_ok() { // The tests succeeded; nothing to do. return; } if !builder.config.cmd.bless() { - crate::detail_exit_macro!(1); + crate::exit!(1); } } } @@ -855,7 +855,7 @@ impl Step for RustdocTheme { util::lld_flag_no_threads(self.compiler.host.contains("windows")), ); } - try_run(builder, &mut cmd).unwrap(); + builder.run_delaying_failure(&mut cmd); } } @@ -870,7 +870,8 @@ impl Step for RustdocJSStd { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("tests/rustdoc-js-std") + let default = run.builder.config.nodejs.is_some(); + run.suite_path("tests/rustdoc-js-std").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -878,38 +879,42 @@ impl Step for RustdocJSStd { } fn run(self, builder: &Builder<'_>) { - if let Some(ref nodejs) = builder.config.nodejs { - let mut command = Command::new(nodejs); - command - .arg(builder.src.join("src/tools/rustdoc-js/tester.js")) - .arg("--crate-name") - .arg("std") - .arg("--resource-suffix") - .arg(&builder.version) - .arg("--doc-folder") - .arg(builder.doc_out(self.target)) - .arg("--test-folder") - .arg(builder.src.join("tests/rustdoc-js-std")); - for path in &builder.paths { - if let Some(p) = - util::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) - { - if !p.ends_with(".js") { - eprintln!("A non-js file was given: `{}`", path.display()); - panic!("Cannot run rustdoc-js-std tests"); - } - command.arg("--test-file").arg(path); + let nodejs = + builder.config.nodejs.as_ref().expect("need nodejs to run rustdoc-js-std tests"); + let mut command = Command::new(nodejs); + command + .arg(builder.src.join("src/tools/rustdoc-js/tester.js")) + .arg("--crate-name") + .arg("std") + .arg("--resource-suffix") + .arg(&builder.version) + .arg("--doc-folder") + .arg(builder.doc_out(self.target)) + .arg("--test-folder") + .arg(builder.src.join("tests/rustdoc-js-std")); + for path in &builder.paths { + if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) { + if !p.ends_with(".js") { + eprintln!("A non-js file was given: `{}`", path.display()); + panic!("Cannot run rustdoc-js-std tests"); } + command.arg("--test-file").arg(path); } - builder.ensure(crate::doc::Std::new( - builder.top_stage, - self.target, - DocumentationFormat::HTML, - )); - builder.run(&mut command); - } else { - builder.info("No nodejs found, skipping \"tests/rustdoc-js-std\" tests"); } + builder.ensure(crate::doc::Std::new( + builder.top_stage, + self.target, + builder, + DocumentationFormat::HTML, + )); + let _guard = builder.msg( + Kind::Test, + builder.top_stage, + "rustdoc-js-std", + builder.config.build, + self.target, + ); + builder.run(&mut command); } } @@ -925,7 +930,8 @@ impl Step for RustdocJSNotStd { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("tests/rustdoc-js") + let default = run.builder.config.nodejs.is_some(); + run.suite_path("tests/rustdoc-js").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -934,18 +940,14 @@ impl Step for RustdocJSNotStd { } fn run(self, builder: &Builder<'_>) { - if builder.config.nodejs.is_some() { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: "js-doc-test", - suite: "rustdoc-js", - path: "tests/rustdoc-js", - compare_mode: None, - }); - } else { - builder.info("No nodejs found, skipping \"tests/rustdoc-js\" tests"); - } + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: "js-doc-test", + suite: "rustdoc-js", + path: "tests/rustdoc-js", + compare_mode: None, + }); } } @@ -1050,6 +1052,13 @@ impl Step for RustdocGUI { } let _time = util::timeit(&builder); + let _guard = builder.msg_sysroot_tool( + Kind::Test, + self.compiler.stage, + "rustdoc-gui", + self.compiler.host, + self.target, + ); crate::render_tests::try_run_tests(builder, &mut cmd, true); } } @@ -1086,6 +1095,14 @@ impl Step for Tidy { if builder.config.cmd.bless() { cmd.arg("--bless"); } + if let Some(s) = builder.config.cmd.extra_checks() { + cmd.arg(format!("--extra-checks={s}")); + } + let mut args = std::env::args_os(); + if let Some(_) = args.find(|arg| arg == OsStr::new("--")) { + cmd.arg("--"); + cmd.args(args); + } if builder.config.channel == "dev" || builder.config.channel == "nightly" { builder.info("fmt check"); @@ -1096,17 +1113,17 @@ impl Step for Tidy { error: no `rustfmt` binary found in {PATH} info: `rust.channel` is currently set to \"{CHAN}\" help: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file -help: to skip test's attempt to check tidiness, pass `--exclude src/tools/tidy` to `x.py test`", +help: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to `x.py test`", PATH = inferred_rustfmt_dir.display(), CHAN = builder.config.channel, ); - crate::detail_exit_macro!(1); + crate::exit!(1); } crate::format::format(&builder, !builder.config.cmd.bless(), &[]); } builder.info("tidy check"); - try_run(builder, &mut cmd).unwrap(); + builder.run_delaying_failure(&mut cmd); builder.ensure(ExpandYamlAnchors); @@ -1123,7 +1140,7 @@ help: to skip test's attempt to check tidiness, pass `--exclude src/tools/tidy` eprintln!( "x.py completions were changed; run `x.py run generate-completions` to update them" ); - crate::detail_exit_macro!(1); + crate::exit!(1); } } } @@ -1151,11 +1168,9 @@ impl Step for ExpandYamlAnchors { /// by the user before committing CI changes. fn run(self, builder: &Builder<'_>) { builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded"); - try_run( - builder, + builder.run_delaying_failure( &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src), - ) - .unwrap(); + ); } fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1435,7 +1450,7 @@ help: to test the compiler, use `--stage 1` instead help: to test the standard library, use `--stage 0 library/std` instead note: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." ); - crate::detail_exit_macro!(1); + crate::exit!(1); } let mut compiler = self.compiler; @@ -1572,6 +1587,8 @@ note: if you're sure you want to do this, please open an issue as to why. In the if let Some(ref nodejs) = builder.config.nodejs { cmd.arg("--nodejs").arg(nodejs); + } else if mode == "js-doc-test" { + panic!("need nodejs to run js-doc-test suite"); } if let Some(ref npm) = builder.config.npm { cmd.arg("--npm").arg(npm); @@ -1642,7 +1659,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the cmd.arg("--run-clang-based-tests-with").arg(clang_exe); } - for exclude in &builder.config.exclude { + for exclude in &builder.config.skip { cmd.arg("--skip"); cmd.arg(&exclude); } @@ -1827,7 +1844,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the #[cfg(feature = "build-metrics")] builder.metrics.begin_test_suite( - crate::metrics::TestSuiteMetadata::Compiletest { + build_helper::metrics::TestSuiteMetadata::Compiletest { suite: suite.into(), mode: mode.into(), compare_mode: None, @@ -1852,7 +1869,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the #[cfg(feature = "build-metrics")] builder.metrics.begin_test_suite( - crate::metrics::TestSuiteMetadata::Compiletest { + build_helper::metrics::TestSuiteMetadata::Compiletest { suite: suite.into(), mode: mode.into(), compare_mode: Some(compare_mode.into()), @@ -1893,14 +1910,6 @@ impl Step for BookTest { /// /// This uses the `rustdoc` that sits next to `compiler`. fn run(self, builder: &Builder<'_>) { - let host = self.compiler.host; - let _guard = builder.msg( - Kind::Test, - self.compiler.stage, - &format!("book {}", self.name), - host, - host, - ); // External docs are different from local because: // - Some books need pre-processing by mdbook before being tested. // - They need to save their state to toolstate. @@ -1943,12 +1952,12 @@ impl BookTest { let _guard = builder.msg( Kind::Test, compiler.stage, - format_args!("rustbook {}", self.path.display()), + format_args!("mdbook {}", self.path.display()), compiler.host, compiler.host, ); let _time = util::timeit(&builder); - let toolstate = if try_run(builder, &mut rustbook_cmd).is_ok() { + let toolstate = if builder.run_delaying_failure(&mut rustbook_cmd) { ToolState::TestPass } else { ToolState::TestFail @@ -1959,8 +1968,12 @@ impl BookTest { /// This runs `rustdoc --test` on all `.md` files in the path. fn run_local_doc(self, builder: &Builder<'_>) { let compiler = self.compiler; + let host = self.compiler.host; - builder.ensure(compile::Std::new(compiler, compiler.host)); + builder.ensure(compile::Std::new(compiler, host)); + + let _guard = + builder.msg(Kind::Test, compiler.stage, &format!("book {}", self.name), host, host); // Do a breadth-first traversal of the `src/doc` directory and just run // tests for all files that end in `*.md` @@ -2074,10 +2087,11 @@ impl Step for ErrorIndex { let mut tool = tool::ErrorIndex::command(builder); tool.arg("markdown").arg(&output); - let _guard = + let guard = builder.msg(Kind::Test, compiler.stage, "error-index", compiler.host, compiler.host); let _time = util::timeit(&builder); builder.run_quiet(&mut tool); + drop(guard); // The tests themselves need to link to std, so make sure it is // available. builder.ensure(compile::Std::new(compiler, compiler.host)); @@ -2106,9 +2120,9 @@ fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> cmd.arg("--test-args").arg(test_args); if builder.config.verbose_tests { - try_run(builder, &mut cmd).is_ok() + builder.run_delaying_failure(&mut cmd) } else { - try_run_quiet(builder, &mut cmd) + builder.run_quiet_delaying_failure(&mut cmd) } } @@ -2134,7 +2148,7 @@ impl Step for RustcGuide { let src = builder.src.join(relative_path); let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - let toolstate = if try_run(builder, rustbook_cmd.arg("linkcheck").arg(&src)).is_ok() { + let toolstate = if builder.run_delaying_failure(rustbook_cmd.arg("linkcheck").arg(&src)) { ToolState::TestPass } else { ToolState::TestFail @@ -2185,11 +2199,12 @@ impl Step for CrateLibrustc { /// Given a `cargo test` subcommand, add the appropriate flags and run it. /// /// Returns whether the test succeeded. -fn run_cargo_test( +fn run_cargo_test<'a>( cargo: impl Into<Command>, libtest_args: &[&str], crates: &[Interned<String>], primary_crate: &str, + description: impl Into<Option<&'a str>>, compiler: Compiler, target: TargetSelection, builder: &Builder<'_>, @@ -2197,10 +2212,13 @@ fn run_cargo_test( let mut cargo = prepare_cargo_test(cargo, libtest_args, crates, primary_crate, compiler, target, builder); let _time = util::timeit(&builder); + let _group = description.into().and_then(|what| { + builder.msg_sysroot_tool(Kind::Test, compiler.stage, what, compiler.host, target) + }); #[cfg(feature = "build-metrics")] builder.metrics.begin_test_suite( - crate::metrics::TestSuiteMetadata::CargoPackage { + build_helper::metrics::TestSuiteMetadata::CargoPackage { crates: crates.iter().map(|c| c.to_string()).collect(), target: target.triple.to_string(), host: compiler.host.triple.to_string(), @@ -2223,9 +2241,11 @@ fn prepare_cargo_test( ) -> Command { let mut cargo = cargo.into(); - // If bless is passed, give downstream crates a way to use it - if builder.config.cmd.bless() { - cargo.env("RUSTC_BLESS", "1"); + // Propegate `--bless` if it has not already been set/unset + // Any tools that want to use this should bless if `RUSTC_BLESS` is set to + // anything other than `0`. + if builder.config.cmd.bless() && !cargo.get_envs().any(|v| v.0 == "RUSTC_BLESS") { + cargo.env("RUSTC_BLESS", "Gesundheit"); } // Pass in some standard flags then iterate over the graph we've discovered @@ -2348,7 +2368,7 @@ impl Step for Crate { // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. // Override it. - if builder.download_rustc() { + if builder.download_rustc() && compiler.stage > 0 { let sysroot = builder .out .join(compiler.host.triple) @@ -2362,14 +2382,16 @@ impl Step for Crate { _ => panic!("can only test libraries"), }; - let _guard = builder.msg( - builder.kind, - compiler.stage, - crate_description(&self.crates), - compiler.host, + run_cargo_test( + cargo, + &[], + &self.crates, + &self.crates[0], + &*crate_description(&self.crates), + compiler, target, + builder, ); - run_cargo_test(cargo, &[], &self.crates, &self.crates[0], compiler, target, builder); } } @@ -2462,18 +2484,12 @@ impl Step for CrateRustdoc { dylib_path.insert(0, PathBuf::from(&*libdir)); cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - let _guard = builder.msg_sysroot_tool( - builder.kind, - compiler.stage, - "rustdoc", - compiler.host, - target, - ); run_cargo_test( cargo, &[], &[INTERNER.intern_str("rustdoc:0.0.0")], "rustdoc", + "rustdoc", compiler, target, builder, @@ -2529,13 +2545,12 @@ impl Step for CrateRustdocJsonTypes { &[] }; - let _guard = - builder.msg(builder.kind, compiler.stage, "rustdoc-json-types", compiler.host, target); run_cargo_test( cargo, libtest_args, &[INTERNER.intern_str("rustdoc-json-types")], "rustdoc-json-types", + "rustdoc-json-types", compiler, target, builder, @@ -2574,7 +2589,7 @@ impl Step for RemoteCopyLibs { builder.ensure(compile::Std::new(compiler, target)); - builder.info(&format!("REMOTE copy libs to emulator ({})", target)); + builder.info(&format!("REMOTE copy libs to emulator ({target})")); let server = builder.ensure(tool::RemoteTestServer { compiler, target }); @@ -2676,6 +2691,10 @@ impl Step for Bootstrap { /// Tests the build system itself. fn run(self, builder: &Builder<'_>) { + let host = builder.config.build; + let compiler = builder.compiler(0, host); + let _guard = builder.msg(Kind::Test, 0, "bootstrap", host, host); + let mut check_bootstrap = Command::new(&builder.python()); check_bootstrap .args(["-m", "unittest", "bootstrap_test.py"]) @@ -2684,10 +2703,8 @@ impl Step for Bootstrap { .current_dir(builder.src.join("src/bootstrap/")); // NOTE: we intentionally don't pass test_args here because the args for unittest and cargo test are mutually incompatible. // Use `python -m unittest` manually if you want to pass arguments. - try_run(builder, &mut check_bootstrap).unwrap(); + builder.run_delaying_failure(&mut check_bootstrap); - let host = builder.config.build; - let compiler = builder.compiler(0, host); let mut cmd = Command::new(&builder.initial_cargo); cmd.arg("test") .current_dir(builder.src.join("src/bootstrap")) @@ -2704,7 +2721,7 @@ impl Step for Bootstrap { } // rustbuild tests are racy on directory creation so just run them one at a time. // Since there's not many this shouldn't be a problem. - run_cargo_test(cmd, &["--test-threads=1"], &[], "bootstrap", compiler, host, builder); + run_cargo_test(cmd, &["--test-threads=1"], &[], "bootstrap", None, compiler, host, builder); } fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -2755,8 +2772,14 @@ impl Step for TierCheck { cargo.arg("--verbose"); } - builder.info("platform support check"); - try_run(builder, &mut cargo.into()).unwrap(); + let _guard = builder.msg( + Kind::Test, + self.compiler.stage, + "platform support check", + self.compiler.host, + self.compiler.host, + ); + builder.run_delaying_failure(&mut cargo.into()); } } @@ -2803,8 +2826,6 @@ impl Step for RustInstaller { /// Ensure the version placeholder replacement tool builds fn run(self, builder: &Builder<'_>) { - builder.info("test rust-installer"); - let bootstrap_host = builder.config.build; let compiler = builder.compiler(0, bootstrap_host); let cargo = tool::prepare_tool_cargo( @@ -2817,7 +2838,15 @@ impl Step for RustInstaller { SourceType::InTree, &[], ); - run_cargo_test(cargo, &[], &[], "installer", compiler, bootstrap_host, builder); + + let _guard = builder.msg( + Kind::Test, + compiler.stage, + "rust-installer", + bootstrap_host, + bootstrap_host, + ); + run_cargo_test(cargo, &[], &[], "installer", None, compiler, bootstrap_host, builder); // We currently don't support running the test.sh script outside linux(?) environments. // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a @@ -2836,7 +2865,7 @@ impl Step for RustInstaller { cmd.env("CARGO", &builder.initial_cargo); cmd.env("RUSTC", &builder.initial_rustc); cmd.env("TMP_DIR", &tmpdir); - try_run(builder, &mut cmd).unwrap(); + builder.run_delaying_failure(&mut cmd); } fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 06c031788..e6d27757a 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -34,6 +34,7 @@ struct ToolBuild { } impl Builder<'_> { + #[track_caller] fn msg_tool( &self, mode: Mode, @@ -107,7 +108,8 @@ impl Step for ToolBuild { ); let mut cargo = Command::from(cargo); - let is_expected = builder.try_run(&mut cargo).is_ok(); + #[allow(deprecated)] // we check this in `is_optional_tool` in a second + let is_expected = builder.config.try_run(&mut cargo).is_ok(); builder.save_toolstate( tool, @@ -116,7 +118,7 @@ impl Step for ToolBuild { if !is_expected { if !is_optional_tool { - crate::detail_exit_macro!(1); + crate::exit!(1); } else { None } @@ -303,6 +305,7 @@ bootstrap_tool!( SuggestTests, "src/tools/suggest-tests", "suggest-tests"; GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys"; RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; + OptimizedDist, "src/tools/opt-dist", "opt-dist"; ); #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] @@ -555,39 +558,6 @@ impl Step for Cargo { allow_features: "", }) .expect("expected to build -- essential tool"); - - let build_cred = |name, path| { - // These credential helpers are currently experimental. - // Any build failures will be ignored. - let _ = builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: name, - mode: Mode::ToolRustc, - path, - is_optional_tool: true, - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - }); - }; - - if self.target.contains("windows") { - build_cred( - "cargo-credential-wincred", - "src/tools/cargo/credential/cargo-credential-wincred", - ); - } - if self.target.contains("apple-darwin") { - build_cred( - "cargo-credential-macos-keychain", - "src/tools/cargo/credential/cargo-credential-macos-keychain", - ); - } - build_cred( - "cargo-credential-1password", - "src/tools/cargo/credential/cargo-credential-1password", - ); cargo_bin_path } } diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs index 9c4d0ea26..308023537 100644 --- a/src/bootstrap/toolstate.rs +++ b/src/bootstrap/toolstate.rs @@ -83,15 +83,15 @@ static NIGHTLY_TOOLS: &[(&str, &str)] = &[ fn print_error(tool: &str, submodule: &str) { eprintln!(); - eprintln!("We detected that this PR updated '{}', but its tests failed.", tool); + eprintln!("We detected that this PR updated '{tool}', but its tests failed."); eprintln!(); - eprintln!("If you do intend to update '{}', please check the error messages above and", tool); + eprintln!("If you do intend to update '{tool}', please check the error messages above and"); eprintln!("commit another update."); eprintln!(); - eprintln!("If you do NOT intend to update '{}', please ensure you did not accidentally", tool); - eprintln!("change the submodule at '{}'. You may ask your reviewer for the", submodule); + eprintln!("If you do NOT intend to update '{tool}', please ensure you did not accidentally"); + eprintln!("change the submodule at '{submodule}'. You may ask your reviewer for the"); eprintln!("proper steps."); - crate::detail_exit_macro!(3); + crate::exit!(3); } fn check_changed_files(toolstates: &HashMap<Box<str>, ToolState>) { @@ -105,8 +105,8 @@ fn check_changed_files(toolstates: &HashMap<Box<str>, ToolState>) { let output = match output { Ok(o) => o, Err(e) => { - eprintln!("Failed to get changed files: {:?}", e); - crate::detail_exit_macro!(1); + eprintln!("Failed to get changed files: {e:?}"); + crate::exit!(1); } }; @@ -114,12 +114,12 @@ fn check_changed_files(toolstates: &HashMap<Box<str>, ToolState>) { for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { let changed = output.lines().any(|l| l.starts_with('M') && l.ends_with(submodule)); - eprintln!("Verifying status of {}...", tool); + eprintln!("Verifying status of {tool}..."); if !changed { continue; } - eprintln!("This PR updated '{}', verifying if status is 'test-pass'...", submodule); + eprintln!("This PR updated '{submodule}', verifying if status is 'test-pass'..."); if toolstates[*tool] != ToolState::TestPass { print_error(tool, submodule); } @@ -172,12 +172,12 @@ impl Step for ToolStateCheck { for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { if !toolstates.contains_key(*tool) { did_error = true; - eprintln!("error: Tool `{}` was not recorded in tool state.", tool); + eprintln!("error: Tool `{tool}` was not recorded in tool state."); } } if did_error { - crate::detail_exit_macro!(1); + crate::exit!(1); } check_changed_files(&toolstates); @@ -190,7 +190,7 @@ impl Step for ToolStateCheck { if state != ToolState::TestPass { if !is_nightly { did_error = true; - eprintln!("error: Tool `{}` should be test-pass but is {}", tool, state); + eprintln!("error: Tool `{tool}` should be test-pass but is {state}"); } else if in_beta_week { let old_state = old_toolstate .iter() @@ -200,17 +200,15 @@ impl Step for ToolStateCheck { if state < old_state { did_error = true; eprintln!( - "error: Tool `{}` has regressed from {} to {} during beta week.", - tool, old_state, state + "error: Tool `{tool}` has regressed from {old_state} to {state} during beta week." ); } else { // This warning only appears in the logs, which most // people won't read. It's mostly here for testing and // debugging. eprintln!( - "warning: Tool `{}` is not test-pass (is `{}`), \ - this should be fixed before beta is branched.", - tool, state + "warning: Tool `{tool}` is not test-pass (is `{state}`), \ + this should be fixed before beta is branched." ); } } @@ -223,7 +221,7 @@ impl Step for ToolStateCheck { } if did_error { - crate::detail_exit_macro!(1); + crate::exit!(1); } if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { @@ -262,6 +260,8 @@ impl Builder<'_> { /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be /// done. The file is updated immediately after this function completes. pub fn save_toolstate(&self, tool: &str, state: ToolState) { + use std::io::Write; + // If we're in a dry run setting we don't want to save toolstates as // that means if we e.g. panic down the line it'll look like we tested // everything (but we actually haven't). @@ -286,7 +286,8 @@ impl Builder<'_> { current_toolstates.insert(tool.into(), state); t!(file.seek(SeekFrom::Start(0))); t!(file.set_len(0)); - t!(serde_json::to_writer(file, ¤t_toolstates)); + t!(serde_json::to_writer(&file, ¤t_toolstates)); + t!(writeln!(file)); // make sure this ends in a newline } } } @@ -320,7 +321,7 @@ fn checkout_toolstate_repo() { Err(_) => false, }; if !success { - panic!("git clone unsuccessful (status: {:?})", status); + panic!("git clone unsuccessful (status: {status:?})"); } } @@ -333,7 +334,7 @@ fn prepare_toolstate_config(token: &str) { Err(_) => false, }; if !success { - panic!("git config key={} value={} failed (status: {:?})", key, value, status); + panic!("git config key={key} value={value} failed (status: {status:?})"); } } @@ -343,7 +344,7 @@ fn prepare_toolstate_config(token: &str) { git_config("user.name", "Rust Toolstate Update"); git_config("credential.helper", "store"); - let credential = format!("https://{}:x-oauth-basic@github.com\n", token,); + let credential = format!("https://{token}:x-oauth-basic@github.com\n",); let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials"); t!(fs::write(&git_credential_path, credential)); } diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index b291584b3..3c4a21434 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -46,9 +46,9 @@ pub use t; /// executable for a particular target. pub fn exe(name: &str, target: TargetSelection) -> String { if target.contains("windows") { - format!("{}.exe", name) + format!("{name}.exe") } else if target.contains("uefi") { - format!("{}.efi", name) + format!("{name}.efi") } else { name.to_string() } @@ -153,16 +153,6 @@ pub fn symlink_dir(config: &Config, original: &Path, link: &Path) -> io::Result< } } -/// The CI environment rustbuild is running in. This mainly affects how the logs -/// are printed. -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub enum CiEnv { - /// Not a CI environment. - None, - /// The GitHub Actions environment, for Linux (including Docker), Windows and macOS builds. - GitHubActions, -} - pub fn forcing_clang_based_tests() -> bool { if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") { match &var.to_string_lossy().to_lowercase()[..] { @@ -171,9 +161,8 @@ pub fn forcing_clang_based_tests() -> bool { other => { // Let's make sure typos don't go unnoticed panic!( - "Unrecognized option '{}' set in \ - RUSTBUILD_FORCE_CLANG_BASED_TESTS", - other + "Unrecognized option '{other}' set in \ + RUSTBUILD_FORCE_CLANG_BASED_TESTS" ) } } @@ -229,7 +218,7 @@ pub fn is_valid_test_suite_arg<'a, P: AsRef<Path>>( pub fn run(cmd: &mut Command, print_cmd_on_fail: bool) { if try_run(cmd, print_cmd_on_fail).is_err() { - crate::detail_exit_macro!(1); + crate::exit!(1); } } @@ -237,15 +226,14 @@ pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { let status = match cmd.status() { Ok(status) => status, Err(e) => { - println!("failed to execute command: {:?}\nerror: {}", cmd, e); + println!("failed to execute command: {cmd:?}\nerror: {e}"); return false; } }; if !status.success() && print_cmd_on_fail { println!( - "\n\ncommand did not execute successfully: {:?}\n\ - expected success, got: {}\n\n", - cmd, status + "\n\ncommand did not execute successfully: {cmd:?}\n\ + expected success, got: {status}\n\n" ); } status.success() @@ -253,14 +241,14 @@ pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { pub fn run_suppressed(cmd: &mut Command) { if !try_run_suppressed(cmd) { - crate::detail_exit_macro!(1); + crate::exit!(1); } } pub fn try_run_suppressed(cmd: &mut Command) -> bool { let output = match cmd.output() { Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)), + Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")), }; if !output.status.success() { println!( @@ -293,7 +281,7 @@ pub fn make(host: &str) -> PathBuf { pub fn output(cmd: &mut Command) -> String { let output = match cmd.stderr(Stdio::inherit()).output() { Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)), + Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")), }; if !output.status.success() { panic!( @@ -308,7 +296,7 @@ pub fn output(cmd: &mut Command) -> String { pub fn output_result(cmd: &mut Command) -> Result<String, String> { let output = match cmd.stderr(Stdio::inherit()).output() { Ok(status) => status, - Err(e) => return Err(format!("failed to run command: {:?}: {}", cmd, e)), + Err(e) => return Err(format!("failed to run command: {cmd:?}: {e}")), }; if !output.status.success() { return Err(format!( @@ -338,7 +326,7 @@ pub fn up_to_date(src: &Path, dst: &Path) -> bool { let threshold = mtime(dst); let meta = match fs::metadata(src) { Ok(meta) => meta, - Err(e) => panic!("source {:?} failed to get metadata: {}", src, e), + Err(e) => panic!("source {src:?} failed to get metadata: {e}"), }; if meta.is_dir() { dir_up_to_date(src, threshold) @@ -503,3 +491,7 @@ pub fn lld_flag_no_threads(is_windows: bool) -> &'static str { }); if is_windows { windows } else { other } } + +pub fn dir_is_empty(dir: &Path) -> bool { + t!(std::fs::read_dir(dir)).next().is_none() +} |