use crate::back::lto::ThinBuffer; use crate::back::profiling::{ selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler, }; use crate::base; use crate::common; use crate::consts; use crate::llvm::{self, DiagnosticInfo, PassManager}; use crate::llvm_util; use crate::type_::Type; use crate::LlvmCodegenBackend; use crate::ModuleLlvm; use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::back::write::{ BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, }; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; use rustc_errors::{FatalError, Handler, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::slice; use std::str; use std::sync::Arc; pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError { match llvm::last_error() { Some(err) => handler.fatal(&format!("{}: {}", msg, err)), None => handler.fatal(msg), } } pub fn write_output_file<'ll>( handler: &rustc_errors::Handler, target: &'ll llvm::TargetMachine, pm: &llvm::PassManager<'ll>, m: &'ll llvm::Module, output: &Path, dwo_output: Option<&Path>, file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, ) -> Result<(), FatalError> { debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); unsafe { let output_c = path_to_c_string(output); let dwo_output_c; let dwo_output_ptr = if let Some(dwo_output) = dwo_output { dwo_output_c = path_to_c_string(dwo_output); dwo_output_c.as_ptr() } else { std::ptr::null() }; let result = llvm::LLVMRustWriteOutputFile( target, pm, m, output_c.as_ptr(), dwo_output_ptr, file_type, ); // Record artifact sizes for self-profiling if result == llvm::LLVMRustResult::Success { let artifact_kind = match file_type { llvm::FileType::ObjectFile => "object_file", llvm::FileType::AssemblyFile => "assembly_file", }; record_artifact_size(self_profiler_ref, artifact_kind, output); if let Some(dwo_file) = dwo_output { record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); } } result.into_result().map_err(|()| { let msg = format!("could not write output to {}", output.display()); llvm_err(handler, &msg) }) } } pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm::TargetMachine { let config = TargetMachineFactoryConfig { split_dwarf_file: None }; // Can't use query system here quite yet because this function is invoked before the query // system/tcx is set up. let features = llvm_util::global_llvm_features(sess, false); target_machine_factory(sess, config::OptLevel::No, &features)(config) .unwrap_or_else(|err| llvm_err(sess.diagnostic(), &err).raise()) } pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { tcx.output_filenames(()).split_dwarf_path( tcx.sess.split_debuginfo(), tcx.sess.opts.unstable_opts.split_dwarf_kind, Some(mod_name), ) } else { None }; let config = TargetMachineFactoryConfig { split_dwarf_file }; target_machine_factory( &tcx.sess, tcx.backend_optimization_level(()), tcx.global_backend_features(()), )(config) .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) } pub fn to_llvm_opt_settings( cfg: config::OptLevel, ) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { use self::config::OptLevel::*; match cfg { No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone), Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone), Default => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone), Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault), SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive), } } fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel { use config::OptLevel::*; match cfg { No => llvm::PassBuilderOptLevel::O0, Less => llvm::PassBuilderOptLevel::O1, Default => llvm::PassBuilderOptLevel::O2, Aggressive => llvm::PassBuilderOptLevel::O3, Size => llvm::PassBuilderOptLevel::Os, SizeMin => llvm::PassBuilderOptLevel::Oz, } } fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel { match relocation_model { RelocModel::Static => llvm::RelocModel::Static, // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra attribute. RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC, RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic, RelocModel::Ropi => llvm::RelocModel::ROPI, RelocModel::Rwpi => llvm::RelocModel::RWPI, RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI, } } pub(crate) fn to_llvm_code_model(code_model: Option) -> llvm::CodeModel { match code_model { Some(CodeModel::Tiny) => llvm::CodeModel::Tiny, Some(CodeModel::Small) => llvm::CodeModel::Small, Some(CodeModel::Kernel) => llvm::CodeModel::Kernel, Some(CodeModel::Medium) => llvm::CodeModel::Medium, Some(CodeModel::Large) => llvm::CodeModel::Large, None => llvm::CodeModel::None, } } pub fn target_machine_factory( sess: &Session, optlvl: config::OptLevel, target_features: &[String], ) -> TargetMachineFactoryFn { let reloc_model = to_llvm_relocation_model(sess.relocation_model()); let (opt_level, _) = to_llvm_opt_settings(optlvl); let use_softfp = sess.opts.cg.soft_float; let ffunction_sections = sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections); let fdata_sections = ffunction_sections; let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names; let code_model = to_llvm_code_model(sess.code_model()); let mut singlethread = sess.target.singlethread; // On the wasm target once the `atomics` feature is enabled that means that // we're no longer single-threaded, or otherwise we don't want LLVM to // lower atomic operations to single-threaded operations. if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) { singlethread = false; } let triple = SmallCStr::new(&sess.target.llvm_target); let cpu = SmallCStr::new(llvm_util::target_cpu(sess)); let features = CString::new(target_features.join(",")).unwrap(); let abi = SmallCStr::new(&sess.target.llvm_abiname); let trap_unreachable = sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; let asm_comments = sess.asm_comments(); let relax_elf_relocations = sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); let use_init_array = !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section); let path_mapping = sess.source_map().path_mapping().clone(); Arc::new(move |config: TargetMachineFactoryConfig| { let split_dwarf_file = path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; let split_dwarf_file = CString::new(split_dwarf_file.to_str().unwrap()).unwrap(); let tm = unsafe { llvm::LLVMRustCreateTargetMachine( triple.as_ptr(), cpu.as_ptr(), features.as_ptr(), abi.as_ptr(), code_model, reloc_model, opt_level, use_softfp, ffunction_sections, fdata_sections, funique_section_names, trap_unreachable, singlethread, asm_comments, emit_stack_size_section, relax_elf_relocations, use_init_array, split_dwarf_file.as_ptr(), ) }; tm.ok_or_else(|| { format!("Could not create LLVM TargetMachine for triple: {}", triple.to_str().unwrap()) }) }) } pub(crate) fn save_temp_bitcode( cgcx: &CodegenContext, module: &ModuleCodegen, name: &str, ) { if !cgcx.save_temps { return; } unsafe { let ext = format!("{}.bc", name); let cgu = Some(&module.name[..]); let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); let cstr = path_to_c_string(&path); let llmod = module.module_llvm.llmod(); llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); } } pub struct DiagnosticHandlers<'a> { data: *mut (&'a CodegenContext, &'a Handler), llcx: &'a llvm::Context, old_handler: Option<&'a llvm::DiagnosticHandler>, } impl<'a> DiagnosticHandlers<'a> { pub fn new( cgcx: &'a CodegenContext, handler: &'a Handler, llcx: &'a llvm::Context, ) -> Self { let remark_passes_all: bool; let remark_passes: Vec; match &cgcx.remark { Passes::All => { remark_passes_all = true; remark_passes = Vec::new(); } Passes::Some(passes) => { remark_passes_all = false; remark_passes = passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect(); } }; let remark_passes: Vec<*const c_char> = remark_passes.iter().map(|name: &CString| name.as_ptr()).collect(); let data = Box::into_raw(Box::new((cgcx, handler))); unsafe { let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); llvm::LLVMRustContextConfigureDiagnosticHandler( llcx, diagnostic_handler, data.cast(), remark_passes_all, remark_passes.as_ptr(), remark_passes.len(), ); DiagnosticHandlers { data, llcx, old_handler } } } } impl<'a> Drop for DiagnosticHandlers<'a> { fn drop(&mut self) { unsafe { llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler); drop(Box::from_raw(self.data)); } } } fn report_inline_asm( cgcx: &CodegenContext, msg: String, level: llvm::DiagnosticLevel, mut cookie: c_uint, source: Option<(String, Vec)>, ) { // In LTO build we may get srcloc values from other crates which are invalid // since they use a different source map. To be safe we just suppress these // in LTO builds. if matches!(cgcx.lto, Lto::Fat | Lto::Thin) { cookie = 0; } let level = match level { llvm::DiagnosticLevel::Error => Level::Error { lint: false }, llvm::DiagnosticLevel::Warning => Level::Warning(None), llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); } unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { if user.is_null() { return; } let (cgcx, diag_handler) = *(user as *const (&CodegenContext, &Handler)); match llvm::diagnostic::Diagnostic::unpack(info) { llvm::diagnostic::InlineAsm(inline) => { report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source); } llvm::diagnostic::Optimization(opt) => { let enabled = match cgcx.remark { Passes::All => true, Passes::Some(ref v) => v.iter().any(|s| *s == opt.pass_name), }; if enabled { diag_handler.note_without_error(&format!( "{}:{}:{}: {}: {}", opt.filename, opt.line, opt.column, opt.pass_name, opt.message, )); } } llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => { let msg = llvm::build_string(|s| { llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); diag_handler.warn(&msg); } llvm::diagnostic::Unsupported(diagnostic_ref) => { let msg = llvm::build_string(|s| { llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); diag_handler.err(&msg); } llvm::diagnostic::UnknownDiagnostic(..) => {} } } fn get_pgo_gen_path(config: &ModuleConfig) -> Option { match config.pgo_gen { SwitchWithOptPath::Enabled(ref opt_dir_path) => { let path = if let Some(dir_path) = opt_dir_path { dir_path.join("default_%m.profraw") } else { PathBuf::from("default_%m.profraw") }; Some(CString::new(format!("{}", path.display())).unwrap()) } SwitchWithOptPath::Disabled => None, } } fn get_pgo_use_path(config: &ModuleConfig) -> Option { config .pgo_use .as_ref() .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) } fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option { config .pgo_sample_use .as_ref() .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) } fn get_instr_profile_output_path(config: &ModuleConfig) -> Option { if config.instrument_coverage { Some(CString::new("default_%m_%p.profraw").unwrap()) } else { None } } pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( cgcx: &CodegenContext, diag_handler: &Handler, module: &ModuleCodegen, config: &ModuleConfig, opt_level: config::OptLevel, opt_stage: llvm::OptStage, ) -> Result<(), FatalError> { let unroll_loops = opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin; let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); let pgo_gen_path = get_pgo_gen_path(config); let pgo_use_path = get_pgo_use_path(config); let pgo_sample_use_path = get_pgo_sample_use_path(config); let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; let instr_profile_output_path = get_instr_profile_output_path(config); // Sanitizer instrumentation is only inserted during the pre-link optimization stage. let sanitizer_options = if !is_lto { Some(llvm::SanitizerOptions { sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS), sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS), sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY), sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY), sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int, sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD), sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS), sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS), }) } else { None }; let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() { Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())) } else { None }; let llvm_selfprofiler = llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() }; let llvm_plugins = config.llvm_plugins.join(","); // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. // We would have to add upstream support for this first, before we can support // config.inline_threshold and our more aggressive default thresholds. let result = llvm::LLVMRustOptimizeWithNewPassManager( module.module_llvm.llmod(), &*module.module_llvm.tm, to_pass_builder_opt_level(opt_level), opt_stage, config.no_prepopulate_passes, config.verify_llvm_ir, using_thin_buffers, config.merge_functions, unroll_loops, config.vectorize_slp, config.vectorize_loop, config.no_builtins, config.emit_lifetime_markers, sanitizer_options.as_ref(), pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_coverage, instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_gcov, pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.debug_info_for_profiling, llvm_selfprofiler, selfprofile_before_pass_callback, selfprofile_after_pass_callback, extra_passes.as_ptr().cast(), extra_passes.len(), llvm_plugins.as_ptr().cast(), llvm_plugins.len(), ); result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) } // Unsafe due to LLVM calls. pub(crate) unsafe fn optimize( cgcx: &CodegenContext, diag_handler: &Handler, module: &ModuleCodegen, config: &ModuleConfig, ) -> Result<(), FatalError> { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; let tm = &*module.module_llvm.tm; let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); let module_name = module.name.clone(); let module_name = Some(&module_name[..]); if let Some(false) = config.new_llvm_pass_manager && llvm_util::get_version() >= (15, 0, 0) { diag_handler.warn( "ignoring `-Z new-llvm-pass-manager=no`, which is no longer supported with LLVM 15", ); } if config.emit_no_opt_bc { let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); let out = path_to_c_string(&out); llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); } if let Some(opt_level) = config.opt_level { if llvm_util::should_use_new_llvm_pass_manager( &config.new_llvm_pass_manager, &cgcx.target_arch, ) { let opt_stage = match cgcx.lto { Lto::Fat => llvm::OptStage::PreLinkFatLTO, Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, _ => llvm::OptStage::PreLinkNoLTO, }; return optimize_with_new_llvm_pass_manager( cgcx, diag_handler, module, config, opt_level, opt_stage, ); } if cgcx.prof.llvm_recording_enabled() { diag_handler .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`"); } // Create the two optimizing pass managers. These mirror what clang // does, and are by populated by LLVM's default PassManagerBuilder. // Each manager has a different set of passes, but they also share // some common passes. let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod); let mpm = llvm::LLVMCreatePassManager(); { let find_pass = |pass_name: &str| { let pass_name = SmallCStr::new(pass_name); llvm::LLVMRustFindAndCreatePass(pass_name.as_ptr()) }; if config.verify_llvm_ir { // Verification should run as the very first pass. llvm::LLVMRustAddPass(fpm, find_pass("verify").unwrap()); } let mut extra_passes = Vec::new(); let mut have_name_anon_globals_pass = false; for pass_name in &config.passes { if pass_name == "lint" { // Linting should also be performed early, directly on the generated IR. llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap()); continue; } if let Some(pass) = find_pass(pass_name) { extra_passes.push(pass); } else { diag_handler.warn(&format!("unknown pass `{}`, ignoring", pass_name)); } if pass_name == "name-anon-globals" { have_name_anon_globals_pass = true; } } // Instrumentation must be inserted before optimization, // otherwise LLVM may optimize some functions away which // breaks llvm-cov. // // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. if config.instrument_gcov { llvm::LLVMRustAddPass(mpm, find_pass("insert-gcov-profiling").unwrap()); } if config.instrument_coverage { llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap()); } if config.debug_info_for_profiling { llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap()); } add_sanitizer_passes(config, &mut extra_passes); // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need // to make sure we run LLVM's NameAnonGlobals pass when emitting bitcode; otherwise // we'll get errors in LLVM. let using_thin_buffers = config.bitcode_needed(); if !config.no_prepopulate_passes { llvm::LLVMAddAnalysisPasses(tm, fpm); llvm::LLVMAddAnalysisPasses(tm, mpm); let opt_level = to_llvm_opt_settings(opt_level).0; let prepare_for_thin_lto = cgcx.lto == Lto::Thin || cgcx.lto == Lto::ThinLocal || (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled()); with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| { llvm::LLVMRustAddLastExtensionPasses( b, extra_passes.as_ptr(), extra_passes.len() as size_t, ); llvm::LLVMRustPassManagerBuilderPopulateFunctionPassManager(b, fpm); llvm::LLVMRustPassManagerBuilderPopulateModulePassManager(b, mpm); }); have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; if using_thin_buffers && !prepare_for_thin_lto { llvm::LLVMRustAddPass(mpm, find_pass("name-anon-globals").unwrap()); have_name_anon_globals_pass = true; } } else { // If we don't use the standard pipeline, directly populate the MPM // with the extra passes. for pass in extra_passes { llvm::LLVMRustAddPass(mpm, pass); } } if using_thin_buffers && !have_name_anon_globals_pass { // As described above, this will probably cause an error in LLVM if config.no_prepopulate_passes { diag_handler.err( "The current compilation is going to use thin LTO buffers \ without running LLVM's NameAnonGlobals pass. \ This will likely cause errors in LLVM. Consider adding \ -C passes=name-anon-globals to the compiler command line.", ); } else { bug!( "We are using thin LTO buffers without running the NameAnonGlobals pass. \ This will likely cause errors in LLVM and should never happen." ); } } } diag_handler.abort_if_errors(); // Finally, run the actual optimization passes { let _timer = cgcx.prof.extra_verbose_generic_activity( "LLVM_module_optimize_function_passes", &*module.name, ); llvm::LLVMRustRunFunctionPassManager(fpm, llmod); } { let _timer = cgcx.prof.extra_verbose_generic_activity( "LLVM_module_optimize_module_passes", &*module.name, ); llvm::LLVMRunPassManager(mpm, llmod); } // Deallocate managers that we're now done with llvm::LLVMDisposePassManager(fpm); llvm::LLVMDisposePassManager(mpm); } Ok(()) } unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) { if config.sanitizer.contains(SanitizerSet::ADDRESS) { let recover = config.sanitizer_recover.contains(SanitizerSet::ADDRESS); passes.push(llvm::LLVMRustCreateAddressSanitizerFunctionPass(recover)); passes.push(llvm::LLVMRustCreateModuleAddressSanitizerPass(recover)); } if config.sanitizer.contains(SanitizerSet::MEMORY) { let track_origins = config.sanitizer_memory_track_origins as c_int; let recover = config.sanitizer_recover.contains(SanitizerSet::MEMORY); passes.push(llvm::LLVMRustCreateMemorySanitizerPass(track_origins, recover)); } if config.sanitizer.contains(SanitizerSet::THREAD) { passes.push(llvm::LLVMRustCreateThreadSanitizerPass()); } if config.sanitizer.contains(SanitizerSet::HWADDRESS) { let recover = config.sanitizer_recover.contains(SanitizerSet::HWADDRESS); passes.push(llvm::LLVMRustCreateHWAddressSanitizerPass(recover)); } } pub(crate) fn link( cgcx: &CodegenContext, diag_handler: &Handler, mut modules: Vec>, ) -> Result, FatalError> { use super::lto::{Linker, ModuleBuffer}; // Sort the modules by name to ensure to ensure deterministic behavior. modules.sort_by(|a, b| a.name.cmp(&b.name)); let (first, elements) = modules.split_first().expect("Bug! modules must contain at least one module."); let mut linker = Linker::new(first.module_llvm.llmod()); for module in elements { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); let buffer = ModuleBuffer::new(module.module_llvm.llmod()); linker.add(buffer.data()).map_err(|()| { let msg = format!("failed to serialize module {:?}", module.name); llvm_err(diag_handler, &msg) })?; } drop(linker); Ok(modules.remove(0)) } pub(crate) unsafe fn codegen( cgcx: &CodegenContext, diag_handler: &Handler, module: ModuleCodegen, config: &ModuleConfig, ) -> Result { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name); { let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; let tm = &*module.module_llvm.tm; let module_name = module.name.clone(); let module_name = Some(&module_name[..]); let handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); if cgcx.msvc_imps_needed { create_msvc_imps(cgcx, llcx, llmod); } // A codegen-specific pass manager is used to generate object // files for an LLVM module. // // Apparently each of these pass managers is a one-shot kind of // thing, so we create a new one for each type of output. The // pass manager passed to the closure should be ensured to not // escape the closure itself, and the manager should only be // used once. unsafe fn with_codegen<'ll, F, R>( tm: &'ll llvm::TargetMachine, llmod: &'ll llvm::Module, no_builtins: bool, f: F, ) -> R where F: FnOnce(&'ll mut PassManager<'ll>) -> R, { let cpm = llvm::LLVMCreatePassManager(); llvm::LLVMAddAnalysisPasses(tm, cpm); llvm::LLVMRustAddLibraryInfo(cpm, llmod, no_builtins); f(cpm) } // Two things to note: // - If object files are just LLVM bitcode we write bitcode, copy it to // the .o file, and delete the bitcode if it wasn't otherwise // requested. // - If we don't have the integrated assembler then we need to emit // asm from LLVM and use `gcc` to create the object file. let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name); let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name); if config.bitcode_needed() { let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); let thin = ThinBuffer::new(llmod, config.emit_thin_lto); let data = thin.data(); if let Some(bitcode_filename) = bc_out.file_name() { cgcx.prof.artifact_size( "llvm_bitcode", bitcode_filename.to_string_lossy(), data.len() as u64, ); } if config.emit_bc || config.emit_obj == EmitObj::Bitcode { let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name); if let Err(e) = fs::write(&bc_out, data) { let msg = format!("failed to write bytecode to {}: {}", bc_out.display(), e); diag_handler.err(&msg); } } if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) { let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data); } } if config.emit_ir { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name); let out = cgcx.output_filenames.temp_path(OutputType::LlvmAssembly, module_name); let out_c = path_to_c_string(&out); extern "C" fn demangle_callback( input_ptr: *const c_char, input_len: size_t, output_ptr: *mut c_char, output_len: size_t, ) -> size_t { let input = unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) }; let Ok(input) = str::from_utf8(input) else { return 0 }; let output = unsafe { slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize) }; let mut cursor = io::Cursor::new(output); let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 }; if write!(cursor, "{:#}", demangled).is_err() { // Possible only if provided buffer is not big enough return 0; } cursor.position() as size_t } let result = llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback); if result == llvm::LLVMRustResult::Success { record_artifact_size(&cgcx.prof, "llvm_ir", &out); } result.into_result().map_err(|()| { let msg = format!("failed to write LLVM IR to {}", out.display()); llvm_err(diag_handler, &msg) })?; } if config.emit_asm { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name); let path = cgcx.output_filenames.temp_path(OutputType::Assembly, module_name); // We can't use the same module for asm and object code output, // because that triggers various errors like invalid IR or broken // binaries. So we must clone the module to produce the asm output // if we are also producing object code. let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj { llvm::LLVMCloneModule(llmod) } else { llmod }; with_codegen(tm, llmod, config.no_builtins, |cpm| { write_output_file( diag_handler, tm, cpm, llmod, &path, None, llvm::FileType::AssemblyFile, &cgcx.prof, ) })?; } match config.emit_obj { EmitObj::ObjectCode(_) => { let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name); let dwo_out = cgcx.output_filenames.temp_path_dwo(module_name); let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) { // Don't change how DWARF is emitted when disabled. (SplitDebuginfo::Off, _) => None, // Don't provide a DWARF object path if split debuginfo is enabled but this is // a platform that doesn't support Split DWARF. _ if !cgcx.target_can_use_split_dwarf => None, // Don't provide a DWARF object path in single mode, sections will be written // into the object as normal but ignored by linker. (_, SplitDwarfKind::Single) => None, // Emit (a subset of the) DWARF into a separate dwarf object file in split // mode. (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), }; with_codegen(tm, llmod, config.no_builtins, |cpm| { write_output_file( diag_handler, tm, cpm, llmod, &obj_out, dwo_out, llvm::FileType::ObjectFile, &cgcx.prof, ) })?; } EmitObj::Bitcode => { debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); if let Err(e) = link_or_copy(&bc_out, &obj_out) { diag_handler.err(&format!("failed to copy bitcode to object file: {}", e)); } if !config.emit_bc { debug!("removing_bitcode {:?}", bc_out); ensure_removed(diag_handler, &bc_out); } } EmitObj::None => {} } drop(handlers); } Ok(module.into_compiled_module( config.emit_obj != EmitObj::None, cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo != SplitDebuginfo::Off && cgcx.split_dwarf_kind == SplitDwarfKind::Split, config.emit_bc, &cgcx.output_filenames, )) } fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec { let mut asm = format!(".section {},\"{}\"\n", section_name, section_flags).into_bytes(); asm.extend_from_slice(b".ascii \""); asm.reserve(data.len()); for &byte in data { if byte == b'\\' || byte == b'"' { asm.push(b'\\'); asm.push(byte); } else if byte < 0x20 || byte >= 0x80 { // Avoid non UTF-8 inline assembly. Use octal escape sequence, because it is fixed // width, while hex escapes will consume following characters. asm.push(b'\\'); asm.push(b'0' + ((byte >> 6) & 0x7)); asm.push(b'0' + ((byte >> 3) & 0x7)); asm.push(b'0' + ((byte >> 0) & 0x7)); } else { asm.push(byte); } } asm.extend_from_slice(b"\"\n"); asm } /// Embed the bitcode of an LLVM module in the LLVM module itself. /// /// This is done primarily for iOS where it appears to be standard to compile C /// code at least with `-fembed-bitcode` which creates two sections in the /// executable: /// /// * __LLVM,__bitcode /// * __LLVM,__cmdline /// /// It appears *both* of these sections are necessary to get the linker to /// recognize what's going on. A suitable cmdline value is taken from the /// target spec. /// /// Furthermore debug/O1 builds don't actually embed bitcode but rather just /// embed an empty section. /// /// Basically all of this is us attempting to follow in the footsteps of clang /// on iOS. See #35968 for lots more info. unsafe fn embed_bitcode( cgcx: &CodegenContext, llcx: &llvm::Context, llmod: &llvm::Module, cmdline: &str, bitcode: &[u8], ) { // We're adding custom sections to the output object file, but we definitely // do not want these custom sections to make their way into the final linked // executable. The purpose of these custom sections is for tooling // surrounding object files to work with the LLVM IR, if necessary. For // example rustc's own LTO will look for LLVM IR inside of the object file // in these sections by default. // // To handle this is a bit different depending on the object file format // used by the backend, broken down into a few different categories: // // * Mach-O - this is for macOS. Inspecting the source code for the native // linker here shows that the `.llvmbc` and `.llvmcmd` sections are // automatically skipped by the linker. In that case there's nothing extra // that we need to do here. // // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and // `.llvmcmd` sections, so there's nothing extra we need to do. // // * COFF - if we don't do anything the linker will by default copy all // these sections to the output artifact, not what we want! To subvert // this we want to flag the sections we inserted here as // `IMAGE_SCN_LNK_REMOVE`. // // * ELF - this is very similar to COFF above. One difference is that these // sections are removed from the output linked artifact when // `--gc-sections` is passed, which we pass by default. If that flag isn't // passed though then these sections will show up in the final output. // Additionally the flag that we need to set here is `SHF_EXCLUDE`. // // Unfortunately, LLVM provides no way to set custom section flags. For ELF // and COFF we emit the sections using module level inline assembly for that // reason (see issue #90326 for historical background). let is_apple = cgcx.opts.target_triple.triple().contains("-ios") || cgcx.opts.target_triple.triple().contains("-darwin") || cgcx.opts.target_triple.triple().contains("-tvos") || cgcx.opts.target_triple.triple().contains("-watchos"); if is_apple || cgcx.opts.target_triple.triple().starts_with("wasm") || cgcx.opts.target_triple.triple().starts_with("asmjs") { // We don't need custom section flags, create LLVM globals. let llconst = common::bytes_in_context(llcx, bitcode); let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), "rustc.embedded.module\0".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); let section = if is_apple { "__LLVM,__bitcode\0" } else { ".llvmbc\0" }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), "rustc.embedded.cmdline\0".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); let section = if is_apple { "__LLVM,__cmdline\0" } else { ".llvmcmd\0" }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); } else { // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); } } pub unsafe fn with_llvm_pmb( llmod: &llvm::Module, config: &ModuleConfig, opt_level: llvm::CodeGenOptLevel, prepare_for_thin_lto: bool, f: &mut dyn FnMut(&llvm::PassManagerBuilder), ) { use std::ptr; // Create the PassManagerBuilder for LLVM. We configure it with // reasonable defaults and prepare it to actually populate the pass // manager. let builder = llvm::LLVMRustPassManagerBuilderCreate(); let opt_size = config.opt_size.map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); let inline_threshold = config.inline_threshold; let pgo_gen_path = get_pgo_gen_path(config); let pgo_use_path = get_pgo_use_path(config); let pgo_sample_use_path = get_pgo_sample_use_path(config); llvm::LLVMRustConfigurePassManagerBuilder( builder, opt_level, config.merge_functions, config.vectorize_slp, config.vectorize_loop, prepare_for_thin_lto, pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), opt_size as c_int, ); llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); // Here we match what clang does (kinda). For O0 we only inline // always-inline functions (but don't add lifetime intrinsics), at O1 we // inline with lifetime intrinsics, and O2+ we add an inliner with a // thresholds copied from clang. match (opt_level, opt_size, inline_threshold) { (.., Some(t)) => { llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, t); } (llvm::CodeGenOptLevel::Aggressive, ..) => { llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 275); } (_, llvm::CodeGenOptSizeDefault, _) => { llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 75); } (_, llvm::CodeGenOptSizeAggressive, _) => { llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 25); } (llvm::CodeGenOptLevel::None, ..) => { llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); } (llvm::CodeGenOptLevel::Less, ..) => { llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); } (llvm::CodeGenOptLevel::Default, ..) => { llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 225); } } f(builder); llvm::LLVMRustPassManagerBuilderDispose(builder); } // Create a `__imp_ = &symbol` global for every public static `symbol`. // This is required to satisfy `dllimport` references to static data in .rlibs // when using MSVC linker. We do this only for data, as linker can fix up // code references on its own. // See #26591, #27438 fn create_msvc_imps( cgcx: &CodegenContext, llcx: &llvm::Context, llmod: &llvm::Module, ) { if !cgcx.msvc_imps_needed { return; } // The x86 ABI seems to require that leading underscores are added to symbol // names, so we need an extra underscore on x86. There's also a leading // '\x01' here which disables LLVM's symbol mangling (e.g., no extra // underscores added in front). let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" }; unsafe { let i8p_ty = Type::i8p_llcx(llcx); let globals = base::iter_globals(llmod) .filter(|&val| { llvm::LLVMRustGetLinkage(val) == llvm::Linkage::ExternalLinkage && llvm::LLVMIsDeclaration(val) == 0 }) .filter_map(|val| { // Exclude some symbols that we know are not Rust symbols. let name = llvm::get_value_name(val); if ignored(name) { None } else { Some((val, name)) } }) .map(move |(val, name)| { let mut imp_name = prefix.as_bytes().to_vec(); imp_name.extend(name); let imp_name = CString::new(imp_name).unwrap(); (imp_name, val) }) .collect::>(); for (imp_name, val) in globals { let imp = llvm::LLVMAddGlobal(llmod, i8p_ty, imp_name.as_ptr().cast()); llvm::LLVMSetInitializer(imp, consts::ptrcast(val, i8p_ty)); llvm::LLVMRustSetLinkage(imp, llvm::Linkage::ExternalLinkage); } } // Use this function to exclude certain symbols from `__imp` generation. fn ignored(symbol_name: &[u8]) -> bool { // These are symbols generated by LLVM's profiling instrumentation symbol_name.starts_with(b"__llvm_profile_") } } fn record_artifact_size( self_profiler_ref: &SelfProfilerRef, artifact_kind: &'static str, path: &Path, ) { // Don't stat the file if we are not going to record its size. if !self_profiler_ref.enabled() { return; } if let Some(artifact_name) = path.file_name() { let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size); } }