diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/build_system')
11 files changed, 741 insertions, 703 deletions
diff --git a/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs b/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs index a081fdaa1..0da27f529 100644 --- a/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs +++ b/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs @@ -1,13 +1,12 @@ use std::path::Path; use super::build_sysroot; -use super::config; use super::path::Dirs; use super::prepare::GitRepo; use super::utils::{spawn_and_wait, CargoProject, Compiler}; use super::SysrootKind; -pub(crate) static ABI_CAFE_REPO: GitRepo = +static ABI_CAFE_REPO: GitRepo = GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe"); static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe"); @@ -17,18 +16,10 @@ pub(crate) fn run( sysroot_kind: SysrootKind, dirs: &Dirs, cg_clif_dylib: &Path, - host_triple: &str, - target_triple: &str, + bootstrap_host_compiler: &Compiler, ) { - if !config::get_bool("testsuite.abi-cafe") { - eprintln!("[SKIP] abi-cafe"); - return; - } - - if host_triple != target_triple { - eprintln!("[SKIP] abi-cafe (cross-compilation not supported)"); - return; - } + ABI_CAFE_REPO.fetch(dirs); + spawn_and_wait(ABI_CAFE.fetch("cargo", &bootstrap_host_compiler.rustc, dirs)); eprintln!("Building sysroot for abi-cafe"); build_sysroot::build_sysroot( @@ -36,15 +27,15 @@ pub(crate) fn run( channel, sysroot_kind, cg_clif_dylib, - host_triple, - target_triple, + bootstrap_host_compiler, + bootstrap_host_compiler.triple.clone(), ); eprintln!("Running abi-cafe"); let pairs = ["rustc_calls_cgclif", "cgclif_calls_rustc", "cgclif_calls_cc", "cc_calls_cgclif"]; - let mut cmd = ABI_CAFE.run(&Compiler::host(), dirs); + let mut cmd = ABI_CAFE.run(bootstrap_host_compiler, dirs); cmd.arg("--"); cmd.arg("--pairs"); cmd.args(pairs); diff --git a/compiler/rustc_codegen_cranelift/build_system/bench.rs b/compiler/rustc_codegen_cranelift/build_system/bench.rs new file mode 100644 index 000000000..a9a851d0a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/bench.rs @@ -0,0 +1,103 @@ +use std::env; +use std::fs; +use std::path::Path; + +use super::path::{Dirs, RelPath}; +use super::prepare::GitRepo; +use super::rustc_info::get_file_name; +use super::utils::{hyperfine_command, spawn_and_wait, CargoProject, Compiler}; + +static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github( + "ebobby", + "simple-raytracer", + "804a7a21b9e673a482797aa289a18ed480e4d813", + "<none>", +); + +// Use a separate target dir for the initial LLVM build to reduce unnecessary recompiles +static SIMPLE_RAYTRACER_LLVM: CargoProject = + CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer_llvm"); + +static SIMPLE_RAYTRACER: CargoProject = + CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer"); + +pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { + benchmark_simple_raytracer(dirs, bootstrap_host_compiler); +} + +fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { + if std::process::Command::new("hyperfine").output().is_err() { + eprintln!("Hyperfine not installed"); + eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine"); + std::process::exit(1); + } + + if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() { + SIMPLE_RAYTRACER_REPO.fetch(dirs); + spawn_and_wait(SIMPLE_RAYTRACER.fetch( + &bootstrap_host_compiler.cargo, + &bootstrap_host_compiler.rustc, + dirs, + )); + } + + eprintln!("[LLVM BUILD] simple-raytracer"); + let build_cmd = SIMPLE_RAYTRACER_LLVM.build(bootstrap_host_compiler, dirs); + spawn_and_wait(build_cmd); + fs::copy( + SIMPLE_RAYTRACER_LLVM + .target_dir(dirs) + .join(&bootstrap_host_compiler.triple) + .join("debug") + .join(get_file_name("main", "bin")), + RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_llvm", "bin")), + ) + .unwrap(); + + let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap(); + + eprintln!("[BENCH COMPILE] ebobby/simple-raytracer"); + let cargo_clif = + RelPath::DIST.to_path(dirs).join(get_file_name("cargo_clif", "bin").replace('_', "-")); + let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs); + let target_dir = SIMPLE_RAYTRACER.target_dir(dirs); + + let clean_cmd = format!( + "RUSTC=rustc cargo clean --manifest-path {manifest_path} --target-dir {target_dir}", + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + let llvm_build_cmd = format!( + "RUSTC=rustc cargo build --manifest-path {manifest_path} --target-dir {target_dir}", + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + let clif_build_cmd = format!( + "RUSTC=rustc {cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}", + cargo_clif = cargo_clif.display(), + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + + let bench_compile = + hyperfine_command(1, bench_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd); + + spawn_and_wait(bench_compile); + + eprintln!("[BENCH RUN] ebobby/simple-raytracer"); + fs::copy( + target_dir.join("debug").join(get_file_name("main", "bin")), + RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_clif", "bin")), + ) + .unwrap(); + + let mut bench_run = hyperfine_command( + 0, + bench_runs, + None, + Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(), + Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(), + ); + bench_run.current_dir(RelPath::BUILD.to_path(dirs)); + spawn_and_wait(bench_run); +} diff --git a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs index fde8ef424..4b740fa2d 100644 --- a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs +++ b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs @@ -3,17 +3,17 @@ use std::path::PathBuf; use super::path::{Dirs, RelPath}; use super::rustc_info::get_file_name; -use super::utils::{is_ci, CargoProject, Compiler}; +use super::utils::{is_ci, is_ci_opt, CargoProject, Compiler}; -static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif"); +pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif"); pub(crate) fn build_backend( dirs: &Dirs, channel: &str, - host_triple: &str, + bootstrap_host_compiler: &Compiler, use_unstable_features: bool, ) -> PathBuf { - let mut cmd = CG_CLIF.build(&Compiler::host(), dirs); + let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs); cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode @@ -25,6 +25,10 @@ pub(crate) fn build_backend( // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway cmd.env("CARGO_BUILD_INCREMENTAL", "false"); + + if !is_ci_opt() { + cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true"); + } } if use_unstable_features { @@ -46,7 +50,7 @@ pub(crate) fn build_backend( CG_CLIF .target_dir(dirs) - .join(host_triple) + .join(&bootstrap_host_compiler.triple) .join(channel) .join(get_file_name("rustc_codegen_cranelift", "dylib")) } diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs index cbbf09b9b..2e04f2c68 100644 --- a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs +++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs @@ -1,31 +1,32 @@ use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{self, Command}; use super::path::{Dirs, RelPath}; -use super::rustc_info::{get_file_name, get_rustc_version, get_wrapper_file_name}; -use super::utils::{spawn_and_wait, try_hard_link, CargoProject, Compiler}; +use super::rustc_info::{get_file_name, get_rustc_version, get_toolchain_name}; +use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler}; use super::SysrootKind; static DIST_DIR: RelPath = RelPath::DIST; static BIN_DIR: RelPath = RelPath::DIST.join("bin"); static LIB_DIR: RelPath = RelPath::DIST.join("lib"); -static RUSTLIB_DIR: RelPath = LIB_DIR.join("rustlib"); pub(crate) fn build_sysroot( dirs: &Dirs, channel: &str, sysroot_kind: SysrootKind, cg_clif_dylib_src: &Path, - host_triple: &str, - target_triple: &str, -) { + bootstrap_host_compiler: &Compiler, + target_triple: String, +) -> Compiler { eprintln!("[BUILD] sysroot {:?}", sysroot_kind); DIST_DIR.ensure_fresh(dirs); BIN_DIR.ensure_exists(dirs); LIB_DIR.ensure_exists(dirs); + let is_native = bootstrap_host_compiler.triple == target_triple; + // Copy the backend let cg_clif_dylib_path = if cfg!(windows) { // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the @@ -35,129 +36,169 @@ pub(crate) fn build_sysroot( LIB_DIR } .to_path(dirs) - .join(get_file_name("rustc_codegen_cranelift", "dylib")); + .join(cg_clif_dylib_src.file_name().unwrap()); try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path); // Build and copy rustc and cargo wrappers + let wrapper_base_name = get_file_name("____", "bin"); + let toolchain_name = get_toolchain_name(); for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] { - let wrapper_name = get_wrapper_file_name(wrapper, "bin"); + let wrapper_name = wrapper_base_name.replace("____", wrapper); - let mut build_cargo_wrapper_cmd = Command::new("rustc"); + let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc); build_cargo_wrapper_cmd + .env("TOOLCHAIN_NAME", toolchain_name.clone()) .arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs"))) .arg("-o") .arg(DIST_DIR.to_path(dirs).join(wrapper_name)) - .arg("-g"); + .arg("-Cstrip=debuginfo"); spawn_and_wait(build_cargo_wrapper_cmd); } - let default_sysroot = super::rustc_info::get_default_sysroot(); + let host = build_sysroot_for_triple( + dirs, + channel, + bootstrap_host_compiler.clone(), + &cg_clif_dylib_path, + sysroot_kind, + ); + host.install_into_sysroot(&DIST_DIR.to_path(dirs)); - let host_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(host_triple).join("lib"); - let target_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(target_triple).join("lib"); - fs::create_dir_all(&host_rustlib_lib).unwrap(); - fs::create_dir_all(&target_rustlib_lib).unwrap(); + if !is_native { + build_sysroot_for_triple( + dirs, + channel, + { + let mut bootstrap_target_compiler = bootstrap_host_compiler.clone(); + bootstrap_target_compiler.triple = target_triple.clone(); + bootstrap_target_compiler.set_cross_linker_and_runner(); + bootstrap_target_compiler + }, + &cg_clif_dylib_path, + sysroot_kind, + ) + .install_into_sysroot(&DIST_DIR.to_path(dirs)); + } - if target_triple == "x86_64-pc-windows-gnu" { - if !default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib").exists() { - eprintln!( - "The x86_64-pc-windows-gnu target needs to be installed first before it is possible \ - to compile a sysroot for it.", - ); - process::exit(1); + // Copy std for the host to the lib dir. This is necessary for the jit mode to find + // libstd. + for lib in host.libs { + let filename = lib.file_name().unwrap().to_str().unwrap(); + if filename.contains("std-") && !filename.contains(".rlib") { + try_hard_link(&lib, LIB_DIR.to_path(dirs).join(lib.file_name().unwrap())); } - for file in fs::read_dir( - default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"), - ) - .unwrap() - { - let file = file.unwrap().path(); - if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") { - continue; // only copy object files - } - try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap())); + } + + let mut target_compiler = { + let dirs: &Dirs = &dirs; + let rustc_clif = + RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustc-clif")); + let rustdoc_clif = + RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustdoc-clif")); + + Compiler { + cargo: bootstrap_host_compiler.cargo.clone(), + rustc: rustc_clif.clone(), + rustdoc: rustdoc_clif.clone(), + rustflags: String::new(), + rustdocflags: String::new(), + triple: target_triple, + runner: vec![], } + }; + if !is_native { + target_compiler.set_cross_linker_and_runner(); } + target_compiler +} - match sysroot_kind { - SysrootKind::None => {} // Nothing to do - SysrootKind::Llvm => { - for file in fs::read_dir( - default_sysroot.join("lib").join("rustlib").join(host_triple).join("lib"), - ) - .unwrap() - { - let file = file.unwrap().path(); - let file_name_str = file.file_name().unwrap().to_str().unwrap(); - if (file_name_str.contains("rustc_") - && !file_name_str.contains("rustc_std_workspace_") - && !file_name_str.contains("rustc_demangle")) - || file_name_str.contains("chalk") - || file_name_str.contains("tracing") - || file_name_str.contains("regex") - { - // These are large crates that are part of the rustc-dev component and are not - // necessary to run regular programs. - continue; - } - try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap())); - } +struct SysrootTarget { + triple: String, + libs: Vec<PathBuf>, +} - if target_triple != host_triple { - for file in fs::read_dir( - default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"), - ) - .unwrap() - { - let file = file.unwrap().path(); - try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap())); - } - } +impl SysrootTarget { + fn install_into_sysroot(&self, sysroot: &Path) { + if self.libs.is_empty() { + return; } - SysrootKind::Clif => { - build_clif_sysroot_for_triple(dirs, channel, host_triple, &cg_clif_dylib_path, None); - - if host_triple != target_triple { - // When cross-compiling it is often necessary to manually pick the right linker - let linker = match target_triple { - "aarch64-unknown-linux-gnu" => Some("aarch64-linux-gnu-gcc"), - "s390x-unknown-linux-gnu" => Some("s390x-linux-gnu-gcc"), - _ => None, - }; - build_clif_sysroot_for_triple( - dirs, - channel, - target_triple, - &cg_clif_dylib_path, - linker, - ); - } - // Copy std for the host to the lib dir. This is necessary for the jit mode to find - // libstd. - for file in fs::read_dir(host_rustlib_lib).unwrap() { - let file = file.unwrap().path(); - let filename = file.file_name().unwrap().to_str().unwrap(); - if filename.contains("std-") && !filename.contains(".rlib") { - try_hard_link(&file, LIB_DIR.to_path(dirs).join(file.file_name().unwrap())); - } - } + let target_rustlib_lib = sysroot.join("lib").join("rustlib").join(&self.triple).join("lib"); + fs::create_dir_all(&target_rustlib_lib).unwrap(); + + for lib in &self.libs { + try_hard_link(lib, target_rustlib_lib.join(lib.file_name().unwrap())); + } + } +} + +pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); +pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot"); +pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version"); +pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src"); +pub(crate) static STANDARD_LIBRARY: CargoProject = + CargoProject::new(&BUILD_SYSROOT, "build_sysroot"); +pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup"); + +#[must_use] +fn build_sysroot_for_triple( + dirs: &Dirs, + channel: &str, + compiler: Compiler, + cg_clif_dylib_path: &Path, + sysroot_kind: SysrootKind, +) -> SysrootTarget { + match sysroot_kind { + SysrootKind::None => build_rtstartup(dirs, &compiler) + .unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }), + SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler), + SysrootKind::Clif => { + build_clif_sysroot_for_triple(dirs, channel, compiler, &cg_clif_dylib_path) } } } -// FIXME move to download/ or dist/ -pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = RelPath::BUILD_SYSROOT.join("rustc_version"); -pub(crate) static SYSROOT_SRC: RelPath = RelPath::BUILD_SYSROOT.join("sysroot_src"); -static STANDARD_LIBRARY: CargoProject = CargoProject::new(&RelPath::BUILD_SYSROOT, "build_sysroot"); +#[must_use] +fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget { + let default_sysroot = super::rustc_info::get_default_sysroot(&compiler.rustc); + let mut target_libs = SysrootTarget { triple: compiler.triple, libs: vec![] }; + + for entry in fs::read_dir( + default_sysroot.join("lib").join("rustlib").join(&target_libs.triple).join("lib"), + ) + .unwrap() + { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + continue; + } + let file = entry.path(); + let file_name_str = file.file_name().unwrap().to_str().unwrap(); + if (file_name_str.contains("rustc_") + && !file_name_str.contains("rustc_std_workspace_") + && !file_name_str.contains("rustc_demangle")) + || file_name_str.contains("chalk") + || file_name_str.contains("tracing") + || file_name_str.contains("regex") + { + // These are large crates that are part of the rustc-dev component and are not + // necessary to run regular programs. + continue; + } + target_libs.libs.push(file); + } + + target_libs +} + +#[must_use] fn build_clif_sysroot_for_triple( dirs: &Dirs, channel: &str, - triple: &str, + mut compiler: Compiler, cg_clif_dylib_path: &Path, - linker: Option<&str>, -) { +) -> SysrootTarget { match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) { Err(e) => { eprintln!("Failed to get rustc version for patched sysroot source: {}", e); @@ -165,7 +206,7 @@ fn build_clif_sysroot_for_triple( process::exit(1); } Ok(source_version) => { - let rustc_version = get_rustc_version(); + let rustc_version = get_rustc_version(&compiler.rustc); if source_version != rustc_version { eprintln!("The patched sysroot source is outdated"); eprintln!("Source version: {}", source_version.trim()); @@ -176,37 +217,42 @@ fn build_clif_sysroot_for_triple( } } - let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(triple).join(channel); + let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; + + if let Some(rtstartup_target_libs) = build_rtstartup(dirs, &compiler) { + rtstartup_target_libs.install_into_sysroot(&RTSTARTUP_SYSROOT.to_path(dirs)); + + target_libs.libs.extend(rtstartup_target_libs.libs); + } + + let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join(channel); if !super::config::get_bool("keep_sysroot") { // Cleanup the deps dir, but keep build scripts and the incremental cache for faster // recompilation as they are not affected by changes in cg_clif. - if build_dir.join("deps").exists() { - fs::remove_dir_all(build_dir.join("deps")).unwrap(); - } + remove_dir_if_exists(&build_dir.join("deps")); } // Build sysroot - let mut rustflags = "-Zforce-unstable-if-unmarked -Cpanic=abort".to_string(); + let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string(); rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap())); - rustflags.push_str(&format!(" --sysroot={}", DIST_DIR.to_path(dirs).to_str().unwrap())); + // Necessary for MinGW to find rsbegin.o and rsend.o + rustflags + .push_str(&format!(" --sysroot={}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap())); if channel == "release" { rustflags.push_str(" -Zmir-opt-level=3"); } - if let Some(linker) = linker { - use std::fmt::Write; - write!(rustflags, " -Clinker={}", linker).unwrap(); - } - let mut compiler = Compiler::with_triple(triple.to_owned()); - compiler.rustflags = rustflags; + compiler.rustflags += &rustflags; let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); if channel == "release" { build_cmd.arg("--release"); } build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); + if compiler.triple.contains("apple") { + build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed"); + } spawn_and_wait(build_cmd); - // Copy all relevant files to the sysroot for entry in fs::read_dir(build_dir.join("deps")).unwrap() { let entry = entry.unwrap(); if let Some(ext) = entry.path().extension() { @@ -216,9 +262,35 @@ fn build_clif_sysroot_for_triple( } else { continue; }; - try_hard_link( - entry.path(), - RUSTLIB_DIR.to_path(dirs).join(triple).join("lib").join(entry.file_name()), - ); + target_libs.libs.push(entry.path()); } + + target_libs +} + +fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> { + if !compiler.triple.ends_with("windows-gnu") { + return None; + } + + RTSTARTUP_SYSROOT.ensure_fresh(dirs); + + let rtstartup_src = SYSROOT_SRC.to_path(dirs).join("library").join("rtstartup"); + let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; + + for file in ["rsbegin", "rsend"] { + let obj = RTSTARTUP_SYSROOT.to_path(dirs).join(format!("{file}.o")); + let mut build_rtstartup_cmd = Command::new(&compiler.rustc); + build_rtstartup_cmd + .arg("--target") + .arg(&compiler.triple) + .arg("--emit=obj") + .arg("-o") + .arg(&obj) + .arg(rtstartup_src.join(format!("{file}.rs"))); + spawn_and_wait(build_rtstartup_cmd); + target_libs.libs.push(obj.clone()); + } + + Some(target_libs) } diff --git a/compiler/rustc_codegen_cranelift/build_system/mod.rs b/compiler/rustc_codegen_cranelift/build_system/mod.rs index 1afc9a55c..8a53acdf7 100644 --- a/compiler/rustc_codegen_cranelift/build_system/mod.rs +++ b/compiler/rustc_codegen_cranelift/build_system/mod.rs @@ -2,9 +2,10 @@ use std::env; use std::path::PathBuf; use std::process; -use self::utils::is_ci; +use self::utils::{is_ci, is_ci_opt, Compiler}; mod abi_cafe; +mod bench; mod build_backend; mod build_sysroot; mod config; @@ -14,31 +15,8 @@ mod rustc_info; mod tests; mod utils; -const USAGE: &str = r#"The build system of cg_clif. - -USAGE: - ./y.rs prepare [--out-dir DIR] - ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] - ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] - -OPTIONS: - --sysroot none|clif|llvm - Which sysroot libraries to use: - `none` will not include any standard library in the sysroot. - `clif` will build the standard library using Cranelift. - `llvm` will use the pre-compiled standard library of rustc which is compiled with LLVM. - - --out-dir DIR - Specify the directory in which the download, build and dist directories are stored. - By default this is the working directory. - - --no-unstable-features - fSome features are not yet ready for production usage. This option will disable these - features. This includes the JIT mode and inline assembly support. -"#; - fn usage() { - eprintln!("{USAGE}"); + eprintln!("{}", include_str!("usage.txt")); } macro_rules! arg_error { @@ -54,6 +32,8 @@ enum Command { Prepare, Build, Test, + AbiCafe, + Bench, } #[derive(Copy, Clone, Debug)] @@ -64,12 +44,19 @@ pub(crate) enum SysrootKind { } pub fn main() { - env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1"); + if env::var("RUST_BACKTRACE").is_err() { + env::set_var("RUST_BACKTRACE", "1"); + } env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1"); if is_ci() { // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway env::set_var("CARGO_BUILD_INCREMENTAL", "false"); + + if !is_ci_opt() { + // Enable the Cranelift verifier + env::set_var("CG_CLIF_ENABLE_VERIFIER", "1"); + } } let mut args = env::args().skip(1); @@ -77,6 +64,8 @@ pub fn main() { Some("prepare") => Command::Prepare, Some("build") => Command::Build, Some("test") => Command::Test, + Some("abi-cafe") => Command::AbiCafe, + Some("bench") => Command::Bench, Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag), Some(command) => arg_error!("Unknown command {}", command), None => { @@ -112,24 +101,16 @@ pub fn main() { } } - let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") { - host_triple - } else if let Some(host_triple) = config::get_value("host") { - host_triple - } else { - rustc_info::get_host_triple() - }; - let target_triple = if let Ok(target_triple) = std::env::var("TARGET_TRIPLE") { - if target_triple != "" { - target_triple - } else { - host_triple.clone() // Empty target triple can happen on GHA - } - } else if let Some(target_triple) = config::get_value("target") { - target_triple - } else { - host_triple.clone() - }; + let bootstrap_host_compiler = Compiler::bootstrap_with_triple( + std::env::var("HOST_TRIPLE") + .ok() + .or_else(|| config::get_value("host")) + .unwrap_or_else(|| rustc_info::get_host_triple()), + ); + let target_triple = std::env::var("TARGET_TRIPLE") + .ok() + .or_else(|| config::get_value("target")) + .unwrap_or_else(|| bootstrap_host_compiler.triple.clone()); // FIXME allow changing the location of these dirs using cli arguments let current_dir = std::env::current_dir().unwrap(); @@ -157,8 +138,15 @@ pub fn main() { process::exit(0); } - let cg_clif_dylib = - build_backend::build_backend(&dirs, channel, &host_triple, use_unstable_features); + env::set_var("RUSTC", "rustc_should_be_set_explicitly"); + env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly"); + + let cg_clif_dylib = build_backend::build_backend( + &dirs, + channel, + &bootstrap_host_compiler, + use_unstable_features, + ); match command { Command::Prepare => { // Handled above @@ -169,28 +157,37 @@ pub fn main() { channel, sysroot_kind, &cg_clif_dylib, - &host_triple, - &target_triple, + &bootstrap_host_compiler, + target_triple.clone(), ); - - abi_cafe::run( + } + Command::AbiCafe => { + if bootstrap_host_compiler.triple != target_triple { + eprintln!("Abi-cafe doesn't support cross-compilation"); + process::exit(1); + } + abi_cafe::run(channel, sysroot_kind, &dirs, &cg_clif_dylib, &bootstrap_host_compiler); + } + Command::Build => { + build_sysroot::build_sysroot( + &dirs, channel, sysroot_kind, - &dirs, &cg_clif_dylib, - &host_triple, - &target_triple, + &bootstrap_host_compiler, + target_triple, ); } - Command::Build => { + Command::Bench => { build_sysroot::build_sysroot( &dirs, channel, sysroot_kind, &cg_clif_dylib, - &host_triple, - &target_triple, + &bootstrap_host_compiler, + target_triple, ); + bench::benchmark(&dirs, &bootstrap_host_compiler); } } } diff --git a/compiler/rustc_codegen_cranelift/build_system/path.rs b/compiler/rustc_codegen_cranelift/build_system/path.rs index e93981f1d..329072300 100644 --- a/compiler/rustc_codegen_cranelift/build_system/path.rs +++ b/compiler/rustc_codegen_cranelift/build_system/path.rs @@ -1,6 +1,8 @@ use std::fs; use std::path::PathBuf; +use super::utils::remove_dir_if_exists; + #[derive(Debug, Clone)] pub(crate) struct Dirs { pub(crate) source_dir: PathBuf, @@ -42,7 +44,6 @@ impl RelPath { pub(crate) const DIST: RelPath = RelPath::Base(PathBase::Dist); pub(crate) const SCRIPTS: RelPath = RelPath::SOURCE.join("scripts"); - pub(crate) const BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); pub(crate) const PATCHES: RelPath = RelPath::SOURCE.join("patches"); pub(crate) const fn join(&'static self, suffix: &'static str) -> RelPath { @@ -62,9 +63,7 @@ impl RelPath { pub(crate) fn ensure_fresh(&self, dirs: &Dirs) { let path = self.to_path(dirs); - if path.exists() { - fs::remove_dir_all(&path).unwrap(); - } + remove_dir_if_exists(&path); fs::create_dir_all(path).unwrap(); } } diff --git a/compiler/rustc_codegen_cranelift/build_system/prepare.rs b/compiler/rustc_codegen_cranelift/build_system/prepare.rs index 8ac67e8f9..50b1b7836 100644 --- a/compiler/rustc_codegen_cranelift/build_system/prepare.rs +++ b/compiler/rustc_codegen_cranelift/build_system/prepare.rs @@ -3,73 +3,51 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use super::build_sysroot::{SYSROOT_RUSTC_VERSION, SYSROOT_SRC}; +use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC}; use super::path::{Dirs, RelPath}; -use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version}; -use super::utils::{copy_dir_recursively, spawn_and_wait, Compiler}; +use super::rustc_info::{get_default_sysroot, get_rustc_version}; +use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait}; pub(crate) fn prepare(dirs: &Dirs) { - if RelPath::DOWNLOAD.to_path(dirs).exists() { - std::fs::remove_dir_all(RelPath::DOWNLOAD.to_path(dirs)).unwrap(); - } - std::fs::create_dir_all(RelPath::DOWNLOAD.to_path(dirs)).unwrap(); + RelPath::DOWNLOAD.ensure_fresh(dirs); - prepare_sysroot(dirs); + spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", "rustc", dirs)); - // FIXME maybe install this only locally? - eprintln!("[INSTALL] hyperfine"); - Command::new("cargo") - .arg("install") - .arg("hyperfine") - .env_remove("CARGO_TARGET_DIR") - .spawn() - .unwrap() - .wait() - .unwrap(); + prepare_sysroot(dirs); + spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", "rustc", dirs)); + spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", "rustc", dirs)); - super::abi_cafe::ABI_CAFE_REPO.fetch(dirs); super::tests::RAND_REPO.fetch(dirs); + spawn_and_wait(super::tests::RAND.fetch("cargo", "rustc", dirs)); super::tests::REGEX_REPO.fetch(dirs); + spawn_and_wait(super::tests::REGEX.fetch("cargo", "rustc", dirs)); super::tests::PORTABLE_SIMD_REPO.fetch(dirs); - super::tests::SIMPLE_RAYTRACER_REPO.fetch(dirs); - - eprintln!("[LLVM BUILD] simple-raytracer"); - let host_compiler = Compiler::host(); - let build_cmd = super::tests::SIMPLE_RAYTRACER.build(&host_compiler, dirs); - spawn_and_wait(build_cmd); - fs::copy( - super::tests::SIMPLE_RAYTRACER - .target_dir(dirs) - .join(&host_compiler.triple) - .join("debug") - .join(get_file_name("main", "bin")), - RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_llvm", "bin")), - ) - .unwrap(); + spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", "rustc", dirs)); } fn prepare_sysroot(dirs: &Dirs) { - let rustc_path = get_rustc_path(); - let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust"); - let sysroot_src = SYSROOT_SRC; - + let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); - sysroot_src.ensure_fresh(dirs); - fs::create_dir_all(sysroot_src.to_path(dirs).join("library")).unwrap(); eprintln!("[COPY] sysroot src"); + + // FIXME ensure builds error out or update the copy if any of the files copied here change + BUILD_SYSROOT.ensure_fresh(dirs); + copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs)); + + fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap(); copy_dir_recursively( &sysroot_src_orig.join("library"), - &sysroot_src.to_path(dirs).join("library"), + &SYSROOT_SRC.to_path(dirs).join("library"), ); - let rustc_version = get_rustc_version(); + let rustc_version = get_rustc_version(Path::new("rustc")); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); eprintln!("[GIT] init"); - init_git_repo(&sysroot_src.to_path(dirs)); + init_git_repo(&SYSROOT_SRC.to_path(dirs)); - apply_patches(dirs, "sysroot", &sysroot_src.to_path(dirs)); + apply_patches(dirs, "sysroot", &SYSROOT_SRC.to_path(dirs)); } pub(crate) struct GitRepo { @@ -98,7 +76,7 @@ impl GitRepo { } } - fn fetch(&self, dirs: &Dirs) { + pub(crate) fn fetch(&self, dirs: &Dirs) { match self.url { GitRepoUrl::Github { user, repo } => { clone_repo_shallow_github( @@ -118,14 +96,14 @@ impl GitRepo { fn clone_repo(download_dir: &Path, repo: &str, rev: &str) { eprintln!("[CLONE] {}", repo); // Ignore exit code as the repo may already have been checked out - Command::new("git").arg("clone").arg(repo).arg(&download_dir).spawn().unwrap().wait().unwrap(); + git_command(None, "clone").arg(repo).arg(download_dir).spawn().unwrap().wait().unwrap(); - let mut clean_cmd = Command::new("git"); - clean_cmd.arg("checkout").arg("--").arg(".").current_dir(&download_dir); + let mut clean_cmd = git_command(download_dir, "checkout"); + clean_cmd.arg("--").arg("."); spawn_and_wait(clean_cmd); - let mut checkout_cmd = Command::new("git"); - checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(download_dir); + let mut checkout_cmd = git_command(download_dir, "checkout"); + checkout_cmd.arg("-q").arg(rev); spawn_and_wait(checkout_cmd); } @@ -149,8 +127,22 @@ fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: // Download zip archive let mut download_cmd = Command::new("curl"); - download_cmd.arg("--location").arg("--output").arg(&archive_file).arg(archive_url); - spawn_and_wait(download_cmd); + download_cmd + .arg("--max-time") + .arg("600") + .arg("-y") + .arg("30") + .arg("-Y") + .arg("10") + .arg("--connect-timeout") + .arg("30") + .arg("--continue-at") + .arg("-") + .arg("--location") + .arg("--output") + .arg(&archive_file) + .arg(archive_url); + retry_spawn_and_wait(5, download_cmd); // Unpack tar archive let mut unpack_cmd = Command::new("tar"); @@ -167,25 +159,16 @@ fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: } fn init_git_repo(repo_dir: &Path) { - let mut git_init_cmd = Command::new("git"); - git_init_cmd.arg("init").arg("-q").current_dir(repo_dir); + let mut git_init_cmd = git_command(repo_dir, "init"); + git_init_cmd.arg("-q"); spawn_and_wait(git_init_cmd); - let mut git_add_cmd = Command::new("git"); - git_add_cmd.arg("add").arg(".").current_dir(repo_dir); + let mut git_add_cmd = git_command(repo_dir, "add"); + git_add_cmd.arg("."); spawn_and_wait(git_add_cmd); - let mut git_commit_cmd = Command::new("git"); - git_commit_cmd - .arg("-c") - .arg("user.name=Dummy") - .arg("-c") - .arg("user.email=dummy@example.com") - .arg("commit") - .arg("-m") - .arg("Initial commit") - .arg("-q") - .current_dir(repo_dir); + let mut git_commit_cmd = git_command(repo_dir, "commit"); + git_commit_cmd.arg("-m").arg("Initial commit").arg("-q"); spawn_and_wait(git_commit_cmd); } @@ -220,16 +203,8 @@ fn apply_patches(dirs: &Dirs, crate_name: &str, target_dir: &Path) { target_dir.file_name().unwrap(), patch.file_name().unwrap() ); - let mut apply_patch_cmd = Command::new("git"); - apply_patch_cmd - .arg("-c") - .arg("user.name=Dummy") - .arg("-c") - .arg("user.email=dummy@example.com") - .arg("am") - .arg(patch) - .arg("-q") - .current_dir(target_dir); + let mut apply_patch_cmd = git_command(target_dir, "am"); + apply_patch_cmd.arg(patch).arg("-q"); spawn_and_wait(apply_patch_cmd); } } diff --git a/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs b/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs index 8e5ab688e..a70453b44 100644 --- a/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs +++ b/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs @@ -1,9 +1,9 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -pub(crate) fn get_rustc_version() -> String { +pub(crate) fn get_rustc_version(rustc: &Path) -> String { let version_info = - Command::new("rustc").stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout; + Command::new(rustc).stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout; String::from_utf8(version_info).unwrap() } @@ -23,6 +23,16 @@ pub(crate) fn get_host_triple() -> String { .to_owned() } +pub(crate) fn get_toolchain_name() -> String { + let active_toolchain = Command::new("rustup") + .stderr(Stdio::inherit()) + .args(&["show", "active-toolchain"]) + .output() + .unwrap() + .stdout; + String::from_utf8(active_toolchain).unwrap().trim().split_once(' ').unwrap().0.to_owned() +} + pub(crate) fn get_cargo_path() -> PathBuf { let cargo_path = Command::new("rustup") .stderr(Stdio::inherit()) @@ -53,8 +63,8 @@ pub(crate) fn get_rustdoc_path() -> PathBuf { Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned() } -pub(crate) fn get_default_sysroot() -> PathBuf { - let default_sysroot = Command::new("rustc") +pub(crate) fn get_default_sysroot(rustc: &Path) -> PathBuf { + let default_sysroot = Command::new(rustc) .stderr(Stdio::inherit()) .args(&["--print", "sysroot"]) .output() @@ -83,12 +93,3 @@ pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String { assert!(file_name.contains(crate_name)); file_name } - -/// Similar to `get_file_name`, but converts any dashes (`-`) in the `crate_name` to -/// underscores (`_`). This is specially made for the rustc and cargo wrappers -/// which have a dash in the name, and that is not allowed in a crate name. -pub(crate) fn get_wrapper_file_name(crate_name: &str, crate_type: &str) -> String { - let crate_name = crate_name.replace('-', "_"); - let wrapper_name = get_file_name(&crate_name, crate_type); - wrapper_name.replace('_', "-") -} diff --git a/compiler/rustc_codegen_cranelift/build_system/tests.rs b/compiler/rustc_codegen_cranelift/build_system/tests.rs index 1c372736e..e9486888f 100644 --- a/compiler/rustc_codegen_cranelift/build_system/tests.rs +++ b/compiler/rustc_codegen_cranelift/build_system/tests.rs @@ -1,11 +1,9 @@ -use super::build_sysroot; +use super::build_sysroot::{self, SYSROOT_SRC}; use super::config; use super::path::{Dirs, RelPath}; use super::prepare::GitRepo; -use super::rustc_info::{get_cargo_path, get_wrapper_file_name}; -use super::utils::{ - hyperfine_command, spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler, -}; +use super::rustc_info::get_host_triple; +use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler}; use super::SysrootKind; use std::env; use std::ffi::OsStr; @@ -17,256 +15,111 @@ static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example"); struct TestCase { config: &'static str, - func: &'static dyn Fn(&TestRunner), + cmd: TestCaseCmd, +} + +enum TestCaseCmd { + Custom { func: &'static dyn Fn(&TestRunner) }, + BuildLib { source: &'static str, crate_types: &'static str }, + BuildBinAndRun { source: &'static str, args: &'static [&'static str] }, + JitBin { source: &'static str, args: &'static str }, } impl TestCase { - const fn new(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self { - Self { config, func } + // FIXME reduce usage of custom test case commands + const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self { + Self { config, cmd: TestCaseCmd::Custom { func } } + } + + const fn build_lib( + config: &'static str, + source: &'static str, + crate_types: &'static str, + ) -> Self { + Self { config, cmd: TestCaseCmd::BuildLib { source, crate_types } } + } + + const fn build_bin_and_run( + config: &'static str, + source: &'static str, + args: &'static [&'static str], + ) -> Self { + Self { config, cmd: TestCaseCmd::BuildBinAndRun { source, args } } + } + + const fn jit_bin(config: &'static str, source: &'static str, args: &'static str) -> Self { + Self { config, cmd: TestCaseCmd::JitBin { source, args } } } } const NO_SYSROOT_SUITE: &[TestCase] = &[ - TestCase::new("build.mini_core", &|runner| { - runner.run_rustc([ - "example/mini_core.rs", - "--crate-name", - "mini_core", - "--crate-type", - "lib,dylib", - "--target", - &runner.target_compiler.triple, - ]); - }), - TestCase::new("build.example", &|runner| { - runner.run_rustc([ - "example/example.rs", - "--crate-type", - "lib", - "--target", - &runner.target_compiler.triple, - ]); - }), - TestCase::new("jit.mini_core_hello_world", &|runner| { - let mut jit_cmd = runner.rustc_command([ - "-Zunstable-options", - "-Cllvm-args=mode=jit", - "-Cprefer-dynamic", - "example/mini_core_hello_world.rs", - "--cfg", - "jit", - "--target", - &runner.target_compiler.triple, - ]); - jit_cmd.env("CG_CLIF_JIT_ARGS", "abc bcd"); - spawn_and_wait(jit_cmd); - - eprintln!("[JIT-lazy] mini_core_hello_world"); - let mut jit_cmd = runner.rustc_command([ - "-Zunstable-options", - "-Cllvm-args=mode=jit-lazy", - "-Cprefer-dynamic", - "example/mini_core_hello_world.rs", - "--cfg", - "jit", - "--target", - &runner.target_compiler.triple, - ]); - jit_cmd.env("CG_CLIF_JIT_ARGS", "abc bcd"); - spawn_and_wait(jit_cmd); - }), - TestCase::new("aot.mini_core_hello_world", &|runner| { - runner.run_rustc([ - "example/mini_core_hello_world.rs", - "--crate-name", - "mini_core_hello_world", - "--crate-type", - "bin", - "-g", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("mini_core_hello_world", ["abc", "bcd"]); - }), + TestCase::build_lib("build.mini_core", "example/mini_core.rs", "lib,dylib"), + TestCase::build_lib("build.example", "example/example.rs", "lib"), + TestCase::jit_bin("jit.mini_core_hello_world", "example/mini_core_hello_world.rs", "abc bcd"), + TestCase::build_bin_and_run( + "aot.mini_core_hello_world", + "example/mini_core_hello_world.rs", + &["abc", "bcd"], + ), ]; const BASE_SYSROOT_SUITE: &[TestCase] = &[ - TestCase::new("aot.arbitrary_self_types_pointers_and_wrappers", &|runner| { - runner.run_rustc([ - "example/arbitrary_self_types_pointers_and_wrappers.rs", - "--crate-name", - "arbitrary_self_types_pointers_and_wrappers", - "--crate-type", - "bin", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("arbitrary_self_types_pointers_and_wrappers", []); - }), - TestCase::new("aot.issue_91827_extern_types", &|runner| { - runner.run_rustc([ - "example/issue-91827-extern-types.rs", - "--crate-name", - "issue_91827_extern_types", - "--crate-type", - "bin", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("issue_91827_extern_types", []); - }), - TestCase::new("build.alloc_system", &|runner| { - runner.run_rustc([ - "example/alloc_system.rs", - "--crate-type", - "lib", - "--target", - &runner.target_compiler.triple, - ]); - }), - TestCase::new("aot.alloc_example", &|runner| { - runner.run_rustc([ - "example/alloc_example.rs", - "--crate-type", - "bin", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("alloc_example", []); - }), - TestCase::new("jit.std_example", &|runner| { - runner.run_rustc([ - "-Zunstable-options", - "-Cllvm-args=mode=jit", - "-Cprefer-dynamic", - "example/std_example.rs", - "--target", - &runner.target_compiler.triple, - ]); - - eprintln!("[JIT-lazy] std_example"); - runner.run_rustc([ - "-Zunstable-options", - "-Cllvm-args=mode=jit-lazy", - "-Cprefer-dynamic", - "example/std_example.rs", - "--target", - &runner.target_compiler.triple, - ]); - }), - TestCase::new("aot.std_example", &|runner| { - runner.run_rustc([ - "example/std_example.rs", - "--crate-type", - "bin", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("std_example", ["arg"]); - }), - TestCase::new("aot.dst_field_align", &|runner| { - runner.run_rustc([ - "example/dst-field-align.rs", - "--crate-name", - "dst_field_align", - "--crate-type", - "bin", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("dst_field_align", []); - }), - TestCase::new("aot.subslice-patterns-const-eval", &|runner| { - runner.run_rustc([ - "example/subslice-patterns-const-eval.rs", - "--crate-type", - "bin", - "-Cpanic=abort", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("subslice-patterns-const-eval", []); - }), - TestCase::new("aot.track-caller-attribute", &|runner| { - runner.run_rustc([ - "example/track-caller-attribute.rs", - "--crate-type", - "bin", - "-Cpanic=abort", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("track-caller-attribute", []); - }), - TestCase::new("aot.float-minmax-pass", &|runner| { - runner.run_rustc([ - "example/float-minmax-pass.rs", - "--crate-type", - "bin", - "-Cpanic=abort", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("float-minmax-pass", []); - }), - TestCase::new("aot.mod_bench", &|runner| { - runner.run_rustc([ - "example/mod_bench.rs", - "--crate-type", - "bin", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("mod_bench", []); - }), - TestCase::new("aot.issue-72793", &|runner| { - runner.run_rustc([ - "example/issue-72793.rs", - "--crate-type", - "bin", - "--target", - &runner.target_compiler.triple, - ]); - runner.run_out_command("issue-72793", []); - }), + TestCase::build_bin_and_run( + "aot.arbitrary_self_types_pointers_and_wrappers", + "example/arbitrary_self_types_pointers_and_wrappers.rs", + &[], + ), + TestCase::build_bin_and_run( + "aot.issue_91827_extern_types", + "example/issue-91827-extern-types.rs", + &[], + ), + TestCase::build_lib("build.alloc_system", "example/alloc_system.rs", "lib"), + TestCase::build_bin_and_run("aot.alloc_example", "example/alloc_example.rs", &[]), + TestCase::jit_bin("jit.std_example", "example/std_example.rs", ""), + TestCase::build_bin_and_run("aot.std_example", "example/std_example.rs", &["arg"]), + TestCase::build_bin_and_run("aot.dst_field_align", "example/dst-field-align.rs", &[]), + TestCase::build_bin_and_run( + "aot.subslice-patterns-const-eval", + "example/subslice-patterns-const-eval.rs", + &[], + ), + TestCase::build_bin_and_run( + "aot.track-caller-attribute", + "example/track-caller-attribute.rs", + &[], + ), + TestCase::build_bin_and_run("aot.float-minmax-pass", "example/float-minmax-pass.rs", &[]), + TestCase::build_bin_and_run("aot.mod_bench", "example/mod_bench.rs", &[]), + TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]), ]; pub(crate) static RAND_REPO: GitRepo = GitRepo::github("rust-random", "rand", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", "rand"); -static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand"); +pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand"); pub(crate) static REGEX_REPO: GitRepo = GitRepo::github("rust-lang", "regex", "341f207c1071f7290e3f228c710817c280c8dca1", "regex"); -static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex"); +pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex"); pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( "rust-lang", "portable-simd", - "d5cd4a8112d958bd3a252327e0d069a6363249bd", + "582239ac3b32007613df04d7ffa78dc30f4c5645", "portable-simd", ); -static PORTABLE_SIMD: CargoProject = +pub(crate) static PORTABLE_SIMD: CargoProject = CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd"); -pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github( - "ebobby", - "simple-raytracer", - "804a7a21b9e673a482797aa289a18ed480e4d813", - "<none>", -); - -pub(crate) static SIMPLE_RAYTRACER: CargoProject = - CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer"); - -static LIBCORE_TESTS: CargoProject = - CargoProject::new(&RelPath::BUILD_SYSROOT.join("sysroot_src/library/core/tests"), "core_tests"); +pub(crate) static LIBCORE_TESTS: CargoProject = + CargoProject::new(&SYSROOT_SRC.join("library/core/tests"), "core_tests"); const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ - TestCase::new("test.rust-random/rand", &|runner| { - spawn_and_wait(RAND.clean(&runner.target_compiler.cargo, &runner.dirs)); + TestCase::custom("test.rust-random/rand", &|runner| { + RAND.clean(&runner.dirs); if runner.is_native { eprintln!("[TEST] rust-random/rand"); @@ -280,60 +133,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ spawn_and_wait(build_cmd); } }), - TestCase::new("bench.simple-raytracer", &|runner| { - let run_runs = env::var("RUN_RUNS").unwrap_or("10".to_string()).parse().unwrap(); - - if runner.is_native { - eprintln!("[BENCH COMPILE] ebobby/simple-raytracer"); - let cargo_clif = RelPath::DIST - .to_path(&runner.dirs) - .join(get_wrapper_file_name("cargo-clif", "bin")); - let manifest_path = SIMPLE_RAYTRACER.manifest_path(&runner.dirs); - let target_dir = SIMPLE_RAYTRACER.target_dir(&runner.dirs); - - let clean_cmd = format!( - "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}", - manifest_path = manifest_path.display(), - target_dir = target_dir.display(), - ); - let llvm_build_cmd = format!( - "cargo build --manifest-path {manifest_path} --target-dir {target_dir}", - manifest_path = manifest_path.display(), - target_dir = target_dir.display(), - ); - let clif_build_cmd = format!( - "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}", - cargo_clif = cargo_clif.display(), - manifest_path = manifest_path.display(), - target_dir = target_dir.display(), - ); - - let bench_compile = - hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd); - - spawn_and_wait(bench_compile); - - eprintln!("[BENCH RUN] ebobby/simple-raytracer"); - fs::copy( - target_dir.join("debug").join("main"), - RelPath::BUILD.to_path(&runner.dirs).join("raytracer_cg_clif"), - ) - .unwrap(); - - let mut bench_run = - hyperfine_command(0, run_runs, None, "./raytracer_cg_llvm", "./raytracer_cg_clif"); - bench_run.current_dir(RelPath::BUILD.to_path(&runner.dirs)); - spawn_and_wait(bench_run); - } else { - spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.target_compiler.cargo, &runner.dirs)); - eprintln!("[BENCH COMPILE] ebobby/simple-raytracer (skipped)"); - eprintln!("[COMPILE] ebobby/simple-raytracer"); - spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs)); - eprintln!("[BENCH RUN] ebobby/simple-raytracer (skipped)"); - } - }), - TestCase::new("test.libcore", &|runner| { - spawn_and_wait(LIBCORE_TESTS.clean(&runner.host_compiler.cargo, &runner.dirs)); + TestCase::custom("test.libcore", &|runner| { + LIBCORE_TESTS.clean(&runner.dirs); if runner.is_native { spawn_and_wait(LIBCORE_TESTS.test(&runner.target_compiler, &runner.dirs)); @@ -344,8 +145,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ spawn_and_wait(build_cmd); } }), - TestCase::new("test.regex-shootout-regex-dna", &|runner| { - spawn_and_wait(REGEX.clean(&runner.target_compiler.cargo, &runner.dirs)); + TestCase::custom("test.regex-shootout-regex-dna", &|runner| { + REGEX.clean(&runner.dirs); // newer aho_corasick versions throw a deprecation warning let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags); @@ -364,9 +165,10 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-input.txt"), ) .unwrap(); - let expected_path = - REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-output.txt"); - let expected = fs::read_to_string(&expected_path).unwrap(); + let expected = fs::read_to_string( + REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-output.txt"), + ) + .unwrap(); let output = spawn_and_wait_with_input(run_cmd, input); // Make sure `[codegen mono items] start` doesn't poison the diff @@ -379,27 +181,16 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ let output_matches = expected.lines().eq(output.lines()); if !output_matches { - let res_path = REGEX.source_dir(&runner.dirs).join("res.txt"); - fs::write(&res_path, &output).unwrap(); - - if cfg!(windows) { - println!("Output files don't match!"); - println!("Expected Output:\n{}", expected); - println!("Actual Output:\n{}", output); - } else { - let mut diff = Command::new("diff"); - diff.arg("-u"); - diff.arg(res_path); - diff.arg(expected_path); - spawn_and_wait(diff); - } + println!("Output files don't match!"); + println!("Expected Output:\n{}", expected); + println!("Actual Output:\n{}", output); std::process::exit(1); } } }), - TestCase::new("test.regex", &|runner| { - spawn_and_wait(REGEX.clean(&runner.host_compiler.cargo, &runner.dirs)); + TestCase::custom("test.regex", &|runner| { + REGEX.clean(&runner.dirs); // newer aho_corasick versions throw a deprecation warning let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags); @@ -425,8 +216,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ spawn_and_wait(build_cmd); } }), - TestCase::new("test.portable-simd", &|runner| { - spawn_and_wait(PORTABLE_SIMD.clean(&runner.host_compiler.cargo, &runner.dirs)); + TestCase::custom("test.portable-simd", &|runner| { + PORTABLE_SIMD.clean(&runner.dirs); let mut build_cmd = PORTABLE_SIMD.build(&runner.target_compiler, &runner.dirs); build_cmd.arg("--all-targets"); @@ -445,21 +236,22 @@ pub(crate) fn run_tests( channel: &str, sysroot_kind: SysrootKind, cg_clif_dylib: &Path, - host_triple: &str, - target_triple: &str, + bootstrap_host_compiler: &Compiler, + target_triple: String, ) { - let runner = TestRunner::new(dirs.clone(), host_triple.to_string(), target_triple.to_string()); - if config::get_bool("testsuite.no_sysroot") { - build_sysroot::build_sysroot( + let target_compiler = build_sysroot::build_sysroot( dirs, channel, SysrootKind::None, cg_clif_dylib, - &host_triple, - &target_triple, + bootstrap_host_compiler, + target_triple.clone(), ); + let runner = + TestRunner::new(dirs.clone(), target_compiler, get_host_triple() == target_triple); + BUILD_EXAMPLE_OUT_DIR.ensure_fresh(dirs); runner.run_testsuite(NO_SYSROOT_SUITE); } else { @@ -470,26 +262,29 @@ pub(crate) fn run_tests( let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot"); if run_base_sysroot || run_extended_sysroot { - build_sysroot::build_sysroot( + let target_compiler = build_sysroot::build_sysroot( dirs, channel, sysroot_kind, cg_clif_dylib, - &host_triple, - &target_triple, + bootstrap_host_compiler, + target_triple.clone(), ); - } - if run_base_sysroot { - runner.run_testsuite(BASE_SYSROOT_SUITE); - } else { - eprintln!("[SKIP] base_sysroot tests"); - } + let runner = + TestRunner::new(dirs.clone(), target_compiler, get_host_triple() == target_triple); - if run_extended_sysroot { - runner.run_testsuite(EXTENDED_SYSROOT_SUITE); - } else { - eprintln!("[SKIP] extended_sysroot tests"); + if run_base_sysroot { + runner.run_testsuite(BASE_SYSROOT_SUITE); + } else { + eprintln!("[SKIP] base_sysroot tests"); + } + + if run_extended_sysroot { + runner.run_testsuite(EXTENDED_SYSROOT_SUITE); + } else { + eprintln!("[SKIP] extended_sysroot tests"); + } } } @@ -497,84 +292,34 @@ struct TestRunner { is_native: bool, jit_supported: bool, dirs: Dirs, - host_compiler: Compiler, target_compiler: Compiler, } impl TestRunner { - pub fn new(dirs: Dirs, host_triple: String, target_triple: String) -> Self { - let is_native = host_triple == target_triple; - let jit_supported = - target_triple.contains("x86_64") && is_native && !host_triple.contains("windows"); - - let rustc_clif = - RelPath::DIST.to_path(&dirs).join(get_wrapper_file_name("rustc-clif", "bin")); - let rustdoc_clif = - RelPath::DIST.to_path(&dirs).join(get_wrapper_file_name("rustdoc-clif", "bin")); - - let mut rustflags = env::var("RUSTFLAGS").ok().unwrap_or("".to_string()); - let mut runner = vec![]; - - if !is_native { - match target_triple.as_str() { - "aarch64-unknown-linux-gnu" => { - // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. - rustflags = format!("-Clinker=aarch64-linux-gnu-gcc{}", rustflags); - runner = vec![ - "qemu-aarch64".to_owned(), - "-L".to_owned(), - "/usr/aarch64-linux-gnu".to_owned(), - ]; - } - "s390x-unknown-linux-gnu" => { - // We are cross-compiling for s390x. Use the correct linker and run tests in qemu. - rustflags = format!("-Clinker=s390x-linux-gnu-gcc{}", rustflags); - runner = vec![ - "qemu-s390x".to_owned(), - "-L".to_owned(), - "/usr/s390x-linux-gnu".to_owned(), - ]; - } - "x86_64-pc-windows-gnu" => { - // We are cross-compiling for Windows. Run tests in wine. - runner = vec!["wine".to_owned()]; - } - _ => { - println!("Unknown non-native platform"); - } - } + pub fn new(dirs: Dirs, mut target_compiler: Compiler, is_native: bool) -> Self { + if let Ok(rustflags) = env::var("RUSTFLAGS") { + target_compiler.rustflags.push(' '); + target_compiler.rustflags.push_str(&rustflags); + } + if let Ok(rustdocflags) = env::var("RUSTDOCFLAGS") { + target_compiler.rustdocflags.push(' '); + target_compiler.rustdocflags.push_str(&rustdocflags); } // FIXME fix `#[linkage = "extern_weak"]` without this - if target_triple.contains("darwin") { - rustflags = format!("{} -Clink-arg=-undefined -Clink-arg=dynamic_lookup", rustflags); + if target_compiler.triple.contains("darwin") { + target_compiler.rustflags.push_str(" -Clink-arg=-undefined -Clink-arg=dynamic_lookup"); } - let host_compiler = Compiler { - cargo: get_cargo_path(), - rustc: rustc_clif.clone(), - rustdoc: rustdoc_clif.clone(), - rustflags: String::new(), - rustdocflags: String::new(), - triple: host_triple, - runner: vec![], - }; - - let target_compiler = Compiler { - cargo: get_cargo_path(), - rustc: rustc_clif, - rustdoc: rustdoc_clif, - rustflags: rustflags.clone(), - rustdocflags: rustflags, - triple: target_triple, - runner, - }; - - Self { is_native, jit_supported, dirs, host_compiler, target_compiler } + let jit_supported = is_native + && target_compiler.triple.contains("x86_64") + && !target_compiler.triple.contains("windows"); + + Self { is_native, jit_supported, dirs, target_compiler } } pub fn run_testsuite(&self, tests: &[TestCase]) { - for &TestCase { config, func } in tests { + for TestCase { config, cmd } in tests { let (tag, testname) = config.split_once('.').unwrap(); let tag = tag.to_uppercase(); let is_jit_test = tag == "JIT"; @@ -586,7 +331,47 @@ impl TestRunner { eprintln!("[{tag}] {testname}"); } - func(self); + match *cmd { + TestCaseCmd::Custom { func } => func(self), + TestCaseCmd::BuildLib { source, crate_types } => { + self.run_rustc([source, "--crate-type", crate_types]); + } + TestCaseCmd::BuildBinAndRun { source, args } => { + self.run_rustc([source]); + self.run_out_command( + source.split('/').last().unwrap().split('.').next().unwrap(), + args, + ); + } + TestCaseCmd::JitBin { source, args } => { + let mut jit_cmd = self.rustc_command([ + "-Zunstable-options", + "-Cllvm-args=mode=jit", + "-Cprefer-dynamic", + source, + "--cfg", + "jit", + ]); + if !args.is_empty() { + jit_cmd.env("CG_CLIF_JIT_ARGS", args); + } + spawn_and_wait(jit_cmd); + + eprintln!("[JIT-lazy] {testname}"); + let mut jit_cmd = self.rustc_command([ + "-Zunstable-options", + "-Cllvm-args=mode=jit-lazy", + "-Cprefer-dynamic", + source, + "--cfg", + "jit", + ]); + if !args.is_empty() { + jit_cmd.env("CG_CLIF_JIT_ARGS", args); + } + spawn_and_wait(jit_cmd); + } + } } } @@ -603,6 +388,9 @@ impl TestRunner { cmd.arg("--out-dir"); cmd.arg(format!("{}", BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).display())); cmd.arg("-Cdebuginfo=2"); + cmd.arg("--target"); + cmd.arg(&self.target_compiler.triple); + cmd.arg("-Cpanic=abort"); cmd.args(args); cmd } @@ -615,10 +403,7 @@ impl TestRunner { spawn_and_wait(self.rustc_command(args)); } - fn run_out_command<'a, I>(&self, name: &str, args: I) - where - I: IntoIterator<Item = &'a str>, - { + fn run_out_command<'a>(&self, name: &str, args: &[&str]) { let mut full_cmd = vec![]; // Prepend the RUN_WRAPPER's @@ -630,7 +415,7 @@ impl TestRunner { BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).join(name).to_str().unwrap().to_string(), ); - for arg in args.into_iter() { + for arg in args { full_cmd.push(arg.to_string()); } diff --git a/compiler/rustc_codegen_cranelift/build_system/usage.txt b/compiler/rustc_codegen_cranelift/build_system/usage.txt new file mode 100644 index 000000000..ab98ccc35 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/usage.txt @@ -0,0 +1,35 @@ +The build system of cg_clif. + +USAGE: + ./y.rs prepare [--out-dir DIR] + ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] + ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] + ./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] + ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] + +OPTIONS: + --debug + Build cg_clif and the standard library in debug mode rather than release mode. + Warning: An unoptimized cg_clif is very slow. + + --sysroot none|clif|llvm + Which sysroot libraries to use: + `none` will not include any standard library in the sysroot. + `clif` will build the standard library using Cranelift. + `llvm` will use the pre-compiled standard library of rustc which is compiled with LLVM. + + --out-dir DIR + Specify the directory in which the download, build and dist directories are stored. + By default this is the working directory. + + --no-unstable-features + Some features are not yet ready for production usage. This option will disable these + features. This includes the JIT mode and inline assembly support. + +REQUIREMENTS: + * Rustup: The build system has a hard coded dependency on rustup to install the right nightly + version and make sure it is used where necessary. + * Git: `./y.rs prepare` uses git for applying patches and on Windows for downloading test repos. + * Curl and tar (non-Windows only): Used by `./y.rs prepare` to download a single commit for + repos. Git will be used to clone the whole repo when using Windows. + * [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.rs bench`. diff --git a/compiler/rustc_codegen_cranelift/build_system/utils.rs b/compiler/rustc_codegen_cranelift/build_system/utils.rs index 2be70e8e4..abc5bab49 100644 --- a/compiler/rustc_codegen_cranelift/build_system/utils.rs +++ b/compiler/rustc_codegen_cranelift/build_system/utils.rs @@ -1,12 +1,13 @@ use std::env; use std::fs; -use std::io::Write; +use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process::{self, Command, Stdio}; use super::path::{Dirs, RelPath}; -use super::rustc_info::{get_cargo_path, get_host_triple, get_rustc_path, get_rustdoc_path}; +use super::rustc_info::{get_cargo_path, get_rustc_path, get_rustdoc_path}; +#[derive(Clone, Debug)] pub(crate) struct Compiler { pub(crate) cargo: PathBuf, pub(crate) rustc: PathBuf, @@ -18,27 +19,47 @@ pub(crate) struct Compiler { } impl Compiler { - pub(crate) fn host() -> Compiler { + pub(crate) fn bootstrap_with_triple(triple: String) -> Compiler { Compiler { cargo: get_cargo_path(), rustc: get_rustc_path(), rustdoc: get_rustdoc_path(), rustflags: String::new(), rustdocflags: String::new(), - triple: get_host_triple(), + triple, runner: vec![], } } - pub(crate) fn with_triple(triple: String) -> Compiler { - Compiler { - cargo: get_cargo_path(), - rustc: get_rustc_path(), - rustdoc: get_rustdoc_path(), - rustflags: String::new(), - rustdocflags: String::new(), - triple, - runner: vec![], + pub(crate) fn set_cross_linker_and_runner(&mut self) { + match self.triple.as_str() { + "aarch64-unknown-linux-gnu" => { + // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. + self.rustflags += " -Clinker=aarch64-linux-gnu-gcc"; + self.rustdocflags += " -Clinker=aarch64-linux-gnu-gcc"; + self.runner = vec![ + "qemu-aarch64".to_owned(), + "-L".to_owned(), + "/usr/aarch64-linux-gnu".to_owned(), + ]; + } + "s390x-unknown-linux-gnu" => { + // We are cross-compiling for s390x. Use the correct linker and run tests in qemu. + self.rustflags += " -Clinker=s390x-linux-gnu-gcc"; + self.rustdocflags += " -Clinker=s390x-linux-gnu-gcc"; + self.runner = vec![ + "qemu-s390x".to_owned(), + "-L".to_owned(), + "/usr/s390x-linux-gnu".to_owned(), + ]; + } + "x86_64-pc-windows-gnu" => { + // We are cross-compiling for Windows. Run tests in wine. + self.runner = vec!["wine".to_owned()]; + } + _ => { + println!("Unknown non-native platform"); + } } } } @@ -65,6 +86,7 @@ impl CargoProject { RelPath::BUILD.join(self.target).to_path(dirs) } + #[must_use] fn base_cmd(&self, command: &str, cargo: &Path, dirs: &Dirs) -> Command { let mut cmd = Command::new(cargo); @@ -72,11 +94,13 @@ impl CargoProject { .arg("--manifest-path") .arg(self.manifest_path(dirs)) .arg("--target-dir") - .arg(self.target_dir(dirs)); + .arg(self.target_dir(dirs)) + .arg("--frozen"); cmd } + #[must_use] fn build_cmd(&self, command: &str, compiler: &Compiler, dirs: &Dirs) -> Command { let mut cmd = self.base_cmd(command, &compiler.cargo, dirs); @@ -97,17 +121,24 @@ impl CargoProject { } #[must_use] - pub(crate) fn fetch(&self, cargo: impl AsRef<Path>, dirs: &Dirs) -> Command { + pub(crate) fn fetch( + &self, + cargo: impl AsRef<Path>, + rustc: impl AsRef<Path>, + dirs: &Dirs, + ) -> Command { let mut cmd = Command::new(cargo.as_ref()); - cmd.arg("fetch").arg("--manifest-path").arg(self.manifest_path(dirs)); + cmd.env("RUSTC", rustc.as_ref()) + .arg("fetch") + .arg("--manifest-path") + .arg(self.manifest_path(dirs)); cmd } - #[must_use] - pub(crate) fn clean(&self, cargo: &Path, dirs: &Dirs) -> Command { - self.base_cmd("clean", cargo, dirs) + pub(crate) fn clean(&self, dirs: &Dirs) { + let _ = fs::remove_dir_all(self.target_dir(dirs)); } #[must_use] @@ -153,6 +184,23 @@ pub(crate) fn hyperfine_command( bench } +#[must_use] +pub(crate) fn git_command<'a>(repo_dir: impl Into<Option<&'a Path>>, cmd: &str) -> Command { + let mut git_cmd = Command::new("git"); + git_cmd + .arg("-c") + .arg("user.name=Dummy") + .arg("-c") + .arg("user.email=dummy@example.com") + .arg("-c") + .arg("core.autocrlf=false") + .arg(cmd); + if let Some(repo_dir) = repo_dir.into() { + git_cmd.current_dir(repo_dir); + } + git_cmd +} + #[track_caller] pub(crate) fn try_hard_link(src: impl AsRef<Path>, dst: impl AsRef<Path>) { let src = src.as_ref(); @@ -169,6 +217,22 @@ pub(crate) fn spawn_and_wait(mut cmd: Command) { } } +// Based on the retry function in rust's src/ci/shared.sh +#[track_caller] +pub(crate) fn retry_spawn_and_wait(tries: u64, mut cmd: Command) { + for i in 1..tries + 1 { + if i != 1 { + println!("Command failed. Attempt {i}/{tries}:"); + } + if cmd.spawn().unwrap().wait().unwrap().success() { + return; + } + std::thread::sleep(std::time::Duration::from_secs(i * 5)); + } + println!("The command has failed after {tries} attempts."); + process::exit(1); +} + #[track_caller] pub(crate) fn spawn_and_wait_with_input(mut cmd: Command, input: String) -> String { let mut child = cmd @@ -190,6 +254,14 @@ pub(crate) fn spawn_and_wait_with_input(mut cmd: Command, input: String) -> Stri String::from_utf8(output.stdout).unwrap() } +pub(crate) fn remove_dir_if_exists(path: &Path) { + match fs::remove_dir_all(&path) { + Ok(()) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => panic!("Failed to remove {path}: {err}", path = path.display()), + } +} + pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) { for entry in fs::read_dir(from).unwrap() { let entry = entry.unwrap(); @@ -207,5 +279,9 @@ pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) { } pub(crate) fn is_ci() -> bool { - env::var("CI").as_deref() == Ok("true") + env::var("CI").is_ok() +} + +pub(crate) fn is_ci_opt() -> bool { + env::var("CI_OPT").is_ok() } |