diff options
Diffstat (limited to 'src/bootstrap/dist.rs')
-rw-r--r-- | src/bootstrap/dist.rs | 227 |
1 files changed, 194 insertions, 33 deletions
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9b2b54961..76aad16c1 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -18,14 +18,16 @@ use std::process::Command; use object::read::archive::ArchiveFile; use object::BinaryFormat; +use sha2::Digest; +use crate::bolt::{instrument_with_bolt, optimize_with_bolt}; use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; use crate::channel; use crate::compile; use crate::config::TargetSelection; use crate::doc::DocumentationFormat; -use crate::native; +use crate::llvm; use crate::tarball::{GeneratedTarball, OverlayKind, Tarball}; use crate::tool::{self, Tool}; use crate::util::{exe, is_dylib, output, t, timeit}; @@ -208,6 +210,8 @@ fn make_win_dist( rustc_dlls.push("libgcc_s_seh-1.dll"); } + // Libraries necessary to link the windows-gnu toolchains. + // System libraries will be preferred if they are available (see #67429). let target_libs = [ //MinGW libs "libgcc.a", @@ -221,6 +225,7 @@ fn make_win_dist( "libmoldname.a", "libpthread.a", //Windows import libs + //This should contain only the set of libraries necessary to link the standard library. "libadvapi32.a", "libbcrypt.a", "libcomctl32.a", @@ -234,6 +239,7 @@ fn make_win_dist( "libkernel32.a", "libmsimg32.a", "libmsvcrt.a", + "libntdll.a", "libodbc32.a", "libole32.a", "liboleaut32.a", @@ -322,7 +328,7 @@ impl Step for Mingw { /// without any extra installed software (e.g., we bundle gcc, libraries, etc). fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let host = self.host; - if !host.ends_with("pc-windows-gnu") { + if !host.ends_with("pc-windows-gnu") || !builder.config.dist_include_mingw_linker { return None; } @@ -378,7 +384,7 @@ impl Step for Rustc { // anything requiring us to distribute a license, but it's likely the // install will *also* include the rust-mingw package, which also needs // licenses, so to be safe we just include it here in all MinGW packages. - if host.ends_with("pc-windows-gnu") { + if host.ends_with("pc-windows-gnu") && builder.config.dist_include_mingw_linker { make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); } @@ -889,6 +895,8 @@ impl Step for Src { /// Creates the `rust-src` installer component fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + builder.update_submodule(&Path::new("src/llvm-project")); + let tarball = Tarball::new_targetless(builder, "rust-src"); // A lot of tools expect the rust-src component to be entirely in this directory, so if you @@ -965,9 +973,10 @@ impl Step for PlainSourceTarball { "RELEASES.md", "configure", "x.py", - "config.toml.example", + "config.example.toml", "Cargo.toml", "Cargo.lock", + ".gitmodules", ]; let src_dirs = ["src", "compiler", "library", "tests"]; @@ -1482,7 +1491,7 @@ impl Step for Extended { let xform = |p: &Path| { let mut contents = t!(fs::read_to_string(p)); - for tool in &["rust-demangler", "miri"] { + for tool in &["rust-demangler", "miri", "rust-docs"] { if !built_tools.contains(tool) { contents = filter(&contents, tool); } @@ -1579,11 +1588,10 @@ impl Step for Extended { prepare("rustc"); prepare("cargo"); prepare("rust-analysis"); - prepare("rust-docs"); prepare("rust-std"); prepare("clippy"); prepare("rust-analyzer"); - for tool in &["rust-demangler", "miri"] { + for tool in &["rust-docs", "rust-demangler", "miri"] { if built_tools.contains(tool) { prepare(tool); } @@ -1618,23 +1626,25 @@ impl Step for Extended { .arg("-out") .arg(exe.join("RustcGroup.wxs")), ); - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-docs") - .args(&heat_flags) - .arg("-cg") - .arg("DocsGroup") - .arg("-dr") - .arg("Docs") - .arg("-var") - .arg("var.DocsDir") - .arg("-out") - .arg(exe.join("DocsGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/squash-components.xsl")), - ); + if built_tools.contains("rust-docs") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-docs") + .args(&heat_flags) + .arg("-cg") + .arg("DocsGroup") + .arg("-dr") + .arg("Docs") + .arg("-var") + .arg("var.DocsDir") + .arg("-out") + .arg(exe.join("DocsGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/squash-components.xsl")), + ); + } builder.run( Command::new(&heat) .current_dir(&exe) @@ -1781,7 +1791,6 @@ impl Step for Extended { cmd.current_dir(&exe) .arg("-nologo") .arg("-dRustcDir=rustc") - .arg("-dDocsDir=rust-docs") .arg("-dCargoDir=cargo") .arg("-dStdDir=rust-std") .arg("-dAnalysisDir=rust-analysis") @@ -1793,6 +1802,9 @@ impl Step for Extended { .arg(&input); add_env(builder, &mut cmd, target); + if built_tools.contains("rust-docs") { + cmd.arg("-dDocsDir=rust-docs"); + } if built_tools.contains("rust-demangler") { cmd.arg("-dRustDemanglerDir=rust-demangler"); } @@ -1811,7 +1823,9 @@ impl Step for Extended { candle(&etc.join("msi/ui.wxs")); candle(&etc.join("msi/rustwelcomedlg.wxs")); candle("RustcGroup.wxs".as_ref()); - candle("DocsGroup.wxs".as_ref()); + if built_tools.contains("rust-docs") { + candle("DocsGroup.wxs".as_ref()); + } candle("CargoGroup.wxs".as_ref()); candle("StdGroup.wxs".as_ref()); candle("ClippyGroup.wxs".as_ref()); @@ -1848,7 +1862,6 @@ impl Step for Extended { .arg("ui.wixobj") .arg("rustwelcomedlg.wixobj") .arg("RustcGroup.wixobj") - .arg("DocsGroup.wixobj") .arg("CargoGroup.wixobj") .arg("StdGroup.wixobj") .arg("AnalysisGroup.wixobj") @@ -1864,6 +1877,9 @@ impl Step for Extended { if built_tools.contains("rust-demangler") { cmd.arg("RustDemanglerGroup.wixobj"); } + if built_tools.contains("rust-docs") { + cmd.arg("DocsGroup.wixobj"); + } if target.ends_with("windows-gnu") { cmd.arg("GccGroup.wixobj"); @@ -1904,6 +1920,26 @@ fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) { } } +fn install_llvm_file(builder: &Builder<'_>, source: &Path, destination: &Path) { + if builder.config.dry_run() { + return; + } + + // After LLVM is built, we modify (instrument or optimize) the libLLVM.so library file. + // This is not done in-place so that the built LLVM files are not "tainted" with BOLT. + // We perform the instrumentation/optimization here, on the fly, just before they are being + // packaged into some destination directory. + let postprocessed = if builder.config.llvm_bolt_profile_generate { + builder.ensure(BoltInstrument::new(source.to_path_buf())) + } else if let Some(path) = &builder.config.llvm_bolt_profile_use { + builder.ensure(BoltOptimize::new(source.to_path_buf(), path.into())) + } else { + source.to_path_buf() + }; + + builder.install(&postprocessed, destination, 0o644); +} + /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking. /// /// Returns whether the files were actually copied. @@ -1927,6 +1963,20 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir } } + // FIXME: for reasons I don't understand, the LLVM so in the `rustc` component is different than the one in `rust-dev`. + // Only the one in `rustc` works with the downloaded compiler. + if builder.download_rustc() && target == builder.build.build { + let src_libdir = builder.ci_rustc_dir(target).join("lib"); + for entry in t!(std::fs::read_dir(&src_libdir)) { + let entry = t!(entry); + if entry.file_name().to_str().unwrap().starts_with("libLLVM-") { + install_llvm_file(builder, &entry.path(), dst_libdir); + return !builder.config.dry_run(); + } + } + panic!("libLLVM.so not found in src_libdir {}!", src_libdir.display()); + } + // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely // clear why this is the case, though. llvm-config will emit the versioned @@ -1939,8 +1989,8 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir builder.install(&llvm_dylib_path, dst_libdir, 0o644); } !builder.config.dry_run() - } else if let Ok(native::LlvmResult { llvm_config, .. }) = - native::prebuilt_llvm_config(builder, target) + } else if let Ok(llvm::LlvmResult { llvm_config, .. }) = + llvm::prebuilt_llvm_config(builder, target) { let mut cmd = Command::new(llvm_config); cmd.arg("--libfiles"); @@ -1955,7 +2005,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir } else { PathBuf::from(file) }; - builder.install(&file, dst_libdir, 0o644); + install_llvm_file(builder, &file, dst_libdir); } !builder.config.dry_run() } else { @@ -1986,6 +2036,117 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection } } +/// Creates an output path to a BOLT-manipulated artifact for the given `file`. +/// The hash of the file is used to make sure that we don't mix BOLT artifacts amongst different +/// files with the same name. +/// +/// We need to keep the file-name the same though, to make sure that copying the manipulated file +/// to a directory will not change the final file path. +fn create_bolt_output_path(builder: &Builder<'_>, file: &Path, hash: &str) -> PathBuf { + let directory = builder.out.join("bolt").join(hash); + t!(fs::create_dir_all(&directory)); + directory.join(file.file_name().unwrap()) +} + +/// Instrument the provided file with BOLT. +/// Returns a path to the instrumented artifact. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BoltInstrument { + file: PathBuf, + hash: String, +} + +impl BoltInstrument { + fn new(file: PathBuf) -> Self { + let mut hasher = sha2::Sha256::new(); + hasher.update(t!(fs::read(&file))); + let hash = hex::encode(hasher.finalize().as_slice()); + + Self { file, hash } + } +} + +impl Step for BoltInstrument { + type Output = PathBuf; + + const ONLY_HOSTS: bool = false; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.build.config.dry_run() { + return self.file.clone(); + } + + if builder.build.config.llvm_from_ci { + println!("warning: trying to use BOLT with LLVM from CI, this will probably not work"); + } + + println!("Instrumenting {} with BOLT", self.file.display()); + + let output_path = create_bolt_output_path(builder, &self.file, &self.hash); + if !output_path.is_file() { + instrument_with_bolt(&self.file, &output_path); + } + output_path + } +} + +/// Optimize the provided file with BOLT. +/// Returns a path to the optimized artifact. +/// +/// The hash is stored in the step to make sure that we don't optimize the same file +/// twice (even under different file paths). +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BoltOptimize { + file: PathBuf, + profile: PathBuf, + hash: String, +} + +impl BoltOptimize { + fn new(file: PathBuf, profile: PathBuf) -> Self { + let mut hasher = sha2::Sha256::new(); + hasher.update(t!(fs::read(&file))); + hasher.update(t!(fs::read(&profile))); + let hash = hex::encode(hasher.finalize().as_slice()); + + Self { file, profile, hash } + } +} + +impl Step for BoltOptimize { + type Output = PathBuf; + + const ONLY_HOSTS: bool = false; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.build.config.dry_run() { + return self.file.clone(); + } + + if builder.build.config.llvm_from_ci { + println!("warning: trying to use BOLT with LLVM from CI, this will probably not work"); + } + + println!("Optimizing {} with BOLT", self.file.display()); + + let output_path = create_bolt_output_path(builder, &self.file, &self.hash); + if !output_path.is_file() { + optimize_with_bolt(&self.file, &self.profile, &output_path); + } + output_path + } +} + #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct LlvmTools { pub target: TargetSelection, @@ -2017,7 +2178,7 @@ impl Step for LlvmTools { } } - builder.ensure(crate::native::Llvm { target }); + builder.ensure(crate::llvm::Llvm { target }); let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); tarball.set_overlay(OverlayKind::LLVM); @@ -2076,10 +2237,10 @@ impl Step for RustDev { let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); tarball.set_overlay(OverlayKind::LLVM); - builder.ensure(crate::native::Llvm { target }); + builder.ensure(crate::llvm::Llvm { target }); // We want to package `lld` to use it with `download-ci-llvm`. - builder.ensure(crate::native::Lld { target }); + builder.ensure(crate::llvm::Lld { target }); let src_bindir = builder.llvm_out(target).join("bin"); // If updating this list, you likely want to change |