diff options
Diffstat (limited to 'compiler/rustc_codegen_ssa')
31 files changed, 1075 insertions, 1081 deletions
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index 0ac12d32b..984efa210 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -48,7 +48,7 @@ libc = "0.2.50" [dependencies.object] version = "0.31.1" default-features = false -features = ["read_core", "elf", "macho", "pe", "unaligned", "archive", "write"] +features = ["read_core", "elf", "macho", "pe", "xcoff", "unaligned", "archive", "write"] [target.'cfg(windows)'.dependencies.windows] version = "0.48.0" diff --git a/compiler/rustc_codegen_ssa/messages.ftl b/compiler/rustc_codegen_ssa/messages.ftl index 9aa2b2e2b..f73080182 100644 --- a/compiler/rustc_codegen_ssa/messages.ftl +++ b/compiler/rustc_codegen_ssa/messages.ftl @@ -9,6 +9,8 @@ codegen_ssa_archive_build_failure = codegen_ssa_atomic_compare_exchange = Atomic compare-exchange intrinsic missing failure memory ordering +codegen_ssa_binary_output_to_tty = option `-o` or `--emit` is used to write binary output type `{$shorthand}` to stdout, but stdout is a tty + codegen_ssa_check_installed_visual_studio = please ensure that Visual Studio 2017 or later, or Build Tools for Visual Studio were installed with the Visual C++ option. codegen_ssa_copy_path = could not copy {$from} to {$to}: {$error} @@ -19,6 +21,8 @@ codegen_ssa_create_temp_dir = couldn't create a temp dir: {$error} codegen_ssa_erroneous_constant = erroneous constant encountered +codegen_ssa_error_creating_remark_dir = failed to create remark directory: {$error} + codegen_ssa_expected_used_symbol = expected `used`, `used(compiler)` or `used(linker)` codegen_ssa_extern_funcs_not_found = some `extern` functions couldn't be found; some native libraries may need to be installed or have their path specified diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 8a00c42a0..b603a8787 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -5,14 +5,14 @@ use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_errors::{ErrorGuaranteed, Handler}; -use rustc_fs_util::fix_windows_verbatim_for_gcc; +use rustc_fs_util::{fix_windows_verbatim_for_gcc, try_canonicalize}; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_metadata::find_native_static_library; -use rustc_metadata::fs::{emit_wrapper_file, METADATA_FILENAME}; +use rustc_metadata::fs::{copy_to_stdout, emit_wrapper_file, METADATA_FILENAME}; use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols::SymbolExportKind; -use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip}; +use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, Strip}; use rustc_session::config::{OutputFilenames, OutputType, PrintRequest, SplitDwarfKind}; use rustc_session::cstore::DllImport; use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename}; @@ -23,7 +23,7 @@ use rustc_session::utils::NativeLibKind; use rustc_session::{filesearch, Session}; use rustc_span::symbol::Symbol; use rustc_target::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault}; -use rustc_target::spec::{Cc, LinkOutputKind, LinkerFlavor, LinkerFlavorCli, Lld, PanicStrategy}; +use rustc_target::spec::{Cc, LinkOutputKind, LinkerFlavor, Lld, PanicStrategy}; use rustc_target::spec::{RelocModel, RelroLevel, SanitizerSet, SplitDebuginfo}; use super::archive::{ArchiveBuilder, ArchiveBuilderBuilder}; @@ -68,6 +68,7 @@ pub fn link_binary<'a>( ) -> Result<(), ErrorGuaranteed> { let _timer = sess.timer("link_binary"); let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata); + let mut tempfiles_for_stdout_output: Vec<PathBuf> = Vec::new(); for &crate_type in sess.crate_types().iter() { // Ignore executable crates if we have -Z no-codegen, as they will error. if (sess.opts.unstable_opts.no_codegen || !sess.opts.output_types.should_codegen()) @@ -97,12 +98,15 @@ pub fn link_binary<'a>( .tempdir() .unwrap_or_else(|error| sess.emit_fatal(errors::CreateTempDir { error })); let path = MaybeTempDir::new(tmpdir, sess.opts.cg.save_temps); - let out_filename = out_filename( + let output = out_filename( sess, crate_type, outputs, codegen_results.crate_info.local_crate_name, ); + let crate_name = format!("{}", codegen_results.crate_info.local_crate_name); + let out_filename = + output.file_for_writing(outputs, OutputType::Exe, Some(crate_name.as_str())); match crate_type { CrateType::Rlib => { let _timer = sess.timer("link_rlib"); @@ -152,6 +156,17 @@ pub fn link_binary<'a>( ); } } + + if output.is_stdout() { + if output.is_tty() { + sess.emit_err(errors::BinaryOutputToTty { + shorthand: OutputType::Exe.shorthand(), + }); + } else if let Err(e) = copy_to_stdout(&out_filename) { + sess.emit_err(errors::CopyPath::new(&out_filename, output.as_path(), e)); + } + tempfiles_for_stdout_output.push(out_filename); + } } } @@ -189,6 +204,11 @@ pub fn link_binary<'a>( remove_temps_from_module(allocator_module); } + // Remove the temporary files if output goes to stdout + for temp in tempfiles_for_stdout_output { + ensure_removed(sess.diagnostic(), &temp); + } + // If no requested outputs require linking, then the object temporaries should // be kept. if !sess.opts.output_types.should_link() { @@ -893,7 +913,7 @@ fn link_natively<'a>( linker_path: &linker_path, exit_status: prog.status, command: &cmd, - escaped_output: &escaped_output, + escaped_output, }; sess.diagnostic().emit_err(err); // If MSVC's `link.exe` was expected but the return code @@ -1188,6 +1208,9 @@ fn add_sanitizer_libraries(sess: &Session, crate_type: CrateType, linker: &mut d if sanitizer.contains(SanitizerSet::HWADDRESS) { link_sanitizer_runtime(sess, linker, "hwasan"); } + if sanitizer.contains(SanitizerSet::SAFESTACK) { + link_sanitizer_runtime(sess, linker, "safestack"); + } } fn link_sanitizer_runtime(sess: &Session, linker: &mut dyn Linker, name: &str) { @@ -1299,44 +1322,7 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| { sess.emit_fatal(errors::LinkerFileStem); }); - - // Remove any version postfix. - let stem = stem - .rsplit_once('-') - .and_then(|(lhs, rhs)| rhs.chars().all(char::is_numeric).then_some(lhs)) - .unwrap_or(stem); - - // GCC/Clang can have an optional target prefix. - let flavor = if stem == "emcc" { - LinkerFlavor::EmCc - } else if stem == "gcc" - || stem.ends_with("-gcc") - || stem == "g++" - || stem.ends_with("-g++") - || stem == "clang" - || stem.ends_with("-clang") - || stem == "clang++" - || stem.ends_with("-clang++") - { - LinkerFlavor::from_cli(LinkerFlavorCli::Gcc, &sess.target) - } else if stem == "wasm-ld" || stem.ends_with("-wasm-ld") { - LinkerFlavor::WasmLld(Cc::No) - } else if stem == "ld" || stem.ends_with("-ld") { - LinkerFlavor::from_cli(LinkerFlavorCli::Ld, &sess.target) - } else if stem == "ld.lld" { - LinkerFlavor::Gnu(Cc::No, Lld::Yes) - } else if stem == "link" { - LinkerFlavor::Msvc(Lld::No) - } else if stem == "lld-link" { - LinkerFlavor::Msvc(Lld::Yes) - } else if stem == "lld" || stem == "rust-lld" { - let lld_flavor = sess.target.linker_flavor.lld_flavor(); - LinkerFlavor::from_cli(LinkerFlavorCli::Lld(lld_flavor), &sess.target) - } else { - // fall back to the value in the target spec - sess.target.linker_flavor - }; - + let flavor = sess.target.linker_flavor.with_linker_hints(stem); Some((linker, flavor)) } (None, None) => None, @@ -1346,7 +1332,7 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { // linker and linker flavor specified via command line have precedence over what the target // specification specifies let linker_flavor = - sess.opts.cg.linker_flavor.map(|flavor| LinkerFlavor::from_cli(flavor, &sess.target)); + sess.opts.cg.linker_flavor.map(|flavor| sess.target.linker_flavor.with_cli_hints(flavor)); if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), linker_flavor) { return ret; } @@ -1702,7 +1688,7 @@ fn detect_self_contained_mingw(sess: &Session) -> bool { /// instead of being found somewhere on the host system. /// We only provide such support for a very limited number of targets. fn self_contained(sess: &Session, crate_type: CrateType) -> bool { - if let Some(self_contained) = sess.opts.cg.link_self_contained { + if let Some(self_contained) = sess.opts.cg.link_self_contained.explicitly_set { if sess.target.link_self_contained == LinkSelfContainedDefault::False { sess.emit_err(errors::UnsupportedLinkSelfContained); } @@ -2131,7 +2117,14 @@ fn linker_with_args<'a>( cmd.add_as_needed(); // Local native libraries of all kinds. - add_local_native_libraries(cmd, sess, archive_builder_builder, codegen_results, tmpdir); + add_local_native_libraries( + cmd, + sess, + archive_builder_builder, + codegen_results, + tmpdir, + link_output_kind, + ); // Upstream rust crates and their non-dynamic native libraries. add_upstream_rust_crates( @@ -2141,10 +2134,18 @@ fn linker_with_args<'a>( codegen_results, crate_type, tmpdir, + link_output_kind, ); // Dynamic native libraries from upstream crates. - add_upstream_native_libraries(cmd, sess, archive_builder_builder, codegen_results, tmpdir); + add_upstream_native_libraries( + cmd, + sess, + archive_builder_builder, + codegen_results, + tmpdir, + link_output_kind, + ); // Link with the import library generated for any raw-dylib functions. for (raw_dylib_name, raw_dylib_imports) in @@ -2245,7 +2246,8 @@ fn add_order_independent_options( out_filename: &Path, tmpdir: &Path, ) { - add_gcc_ld_path(cmd, sess, flavor); + // Take care of the flavors and CLI options requesting the `lld` linker. + add_lld_args(cmd, sess, flavor); add_apple_sdk(cmd, sess, flavor); @@ -2290,11 +2292,13 @@ fn add_order_independent_options( } else if flavor == LinkerFlavor::Bpf { cmd.arg("--cpu"); cmd.arg(&codegen_results.crate_info.target_cpu); - cmd.arg("--cpu-features"); - cmd.arg(match &sess.opts.cg.target_feature { - feat if !feat.is_empty() => feat.as_ref(), - _ => sess.target.options.features.as_ref(), - }); + if let Some(feat) = [sess.opts.cg.target_feature.as_str(), &sess.target.options.features] + .into_iter() + .find(|feat| !feat.is_empty()) + { + cmd.arg("--cpu-features"); + cmd.arg(feat); + } } cmd.linker_plugin_lto(); @@ -2399,6 +2403,7 @@ fn add_native_libs_from_crate( cnum: CrateNum, link_static: bool, link_dynamic: bool, + link_output_kind: LinkOutputKind, ) { if !sess.opts.unstable_opts.link_native_libraries { // If `-Zlink-native-libraries=false` is set, then the assumption is that an @@ -2478,8 +2483,16 @@ fn add_native_libs_from_crate( } } NativeLibKind::Unspecified => { - if link_dynamic { - cmd.link_dylib(name, verbatim, true); + // If we are generating a static binary, prefer static library when the + // link kind is unspecified. + if !link_output_kind.can_link_dylib() && !sess.target.crt_static_allows_dylibs { + if link_static { + cmd.link_staticlib(name, verbatim) + } + } else { + if link_dynamic { + cmd.link_dylib(name, verbatim, true); + } } } NativeLibKind::Framework { as_needed } => { @@ -2506,6 +2519,7 @@ fn add_local_native_libraries( archive_builder_builder: &dyn ArchiveBuilderBuilder, codegen_results: &CodegenResults, tmpdir: &Path, + link_output_kind: LinkOutputKind, ) { if sess.opts.unstable_opts.link_native_libraries { // User-supplied library search paths (-L on the command line). These are the same paths @@ -2535,6 +2549,7 @@ fn add_local_native_libraries( LOCAL_CRATE, link_static, link_dynamic, + link_output_kind, ); } @@ -2545,6 +2560,7 @@ fn add_upstream_rust_crates<'a>( codegen_results: &CodegenResults, crate_type: CrateType, tmpdir: &Path, + link_output_kind: LinkOutputKind, ) { // All of the heavy lifting has previously been accomplished by the // dependency_format module of the compiler. This is just crawling the @@ -2622,6 +2638,7 @@ fn add_upstream_rust_crates<'a>( cnum, link_static, link_dynamic, + link_output_kind, ); } } @@ -2632,6 +2649,7 @@ fn add_upstream_native_libraries( archive_builder_builder: &dyn ArchiveBuilderBuilder, codegen_results: &CodegenResults, tmpdir: &Path, + link_output_kind: LinkOutputKind, ) { let search_path = OnceCell::new(); for &cnum in &codegen_results.crate_info.used_crates { @@ -2660,10 +2678,35 @@ fn add_upstream_native_libraries( cnum, link_static, link_dynamic, + link_output_kind, ); } } +// Rehome lib paths (which exclude the library file name) that point into the sysroot lib directory +// to be relative to the sysroot directory, which may be a relative path specified by the user. +// +// If the sysroot is a relative path, and the sysroot libs are specified as an absolute path, the +// linker command line can be non-deterministic due to the paths including the current working +// directory. The linker command line needs to be deterministic since it appears inside the PDB +// file generated by the MSVC linker. See https://github.com/rust-lang/rust/issues/112586. +// +// The returned path will always have `fix_windows_verbatim_for_gcc()` applied to it. +fn rehome_sysroot_lib_dir<'a>(sess: &'a Session, lib_dir: &Path) -> PathBuf { + let sysroot_lib_path = sess.target_filesearch(PathKind::All).get_lib_path(); + let canonical_sysroot_lib_path = + { try_canonicalize(&sysroot_lib_path).unwrap_or_else(|_| sysroot_lib_path.clone()) }; + + let canonical_lib_dir = try_canonicalize(lib_dir).unwrap_or_else(|_| lib_dir.to_path_buf()); + if canonical_lib_dir == canonical_sysroot_lib_path { + // This path, returned by `target_filesearch().get_lib_path()`, has + // already had `fix_windows_verbatim_for_gcc()` applied if needed. + sysroot_lib_path + } else { + fix_windows_verbatim_for_gcc(&lib_dir) + } +} + // Adds the static "rlib" versions of all crates to the command line. // There's a bit of magic which happens here specifically related to LTO, // namely that we remove upstream object files. @@ -2695,7 +2738,13 @@ fn add_static_crate<'a>( let cratepath = &src.rlib.as_ref().unwrap().0; let mut link_upstream = |path: &Path| { - cmd.link_rlib(&fix_windows_verbatim_for_gcc(path)); + let rlib_path = if let Some(dir) = path.parent() { + let file_name = path.file_name().expect("rlib path has no file name path component"); + rehome_sysroot_lib_dir(sess, &dir).join(file_name) + } else { + fix_windows_verbatim_for_gcc(path) + }; + cmd.link_rlib(&rlib_path); }; if !are_upstream_rust_objects_already_included(sess) @@ -2764,7 +2813,7 @@ fn add_dynamic_crate(cmd: &mut dyn Linker, sess: &Session, cratepath: &Path) { // what its name is let parent = cratepath.parent(); if let Some(dir) = parent { - cmd.include_path(&fix_windows_verbatim_for_gcc(dir)); + cmd.include_path(&rehome_sysroot_lib_dir(sess, dir)); } let stem = cratepath.file_stem().unwrap().to_str().unwrap(); // Convert library file-stem into a cc -l argument. @@ -2900,55 +2949,81 @@ fn get_apple_sdk_root(sdk_name: &str) -> Result<String, errors::AppleSdkRootErro } } -fn add_gcc_ld_path(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { - if let Some(ld_impl) = sess.opts.unstable_opts.gcc_ld { - if let LinkerFlavor::Gnu(Cc::Yes, _) - | LinkerFlavor::Darwin(Cc::Yes, _) - | LinkerFlavor::WasmLld(Cc::Yes) = flavor - { - match ld_impl { - LdImpl::Lld => { - // Implement the "self-contained" part of -Zgcc-ld - // by adding rustc distribution directories to the tool search path. - for path in sess.get_tools_search_paths(false) { - cmd.arg({ - let mut arg = OsString::from("-B"); - arg.push(path.join("gcc-ld")); - arg - }); - } - // Implement the "linker flavor" part of -Zgcc-ld - // by asking cc to use some kind of lld. - cmd.arg("-fuse-ld=lld"); - - if !flavor.is_gnu() { - // Tell clang to use a non-default LLD flavor. - // Gcc doesn't understand the target option, but we currently assume - // that gcc is not used for Apple and Wasm targets (#97402). - // - // Note that we don't want to do that by default on macOS: e.g. passing a - // 10.7 target to LLVM works, but not to recent versions of clang/macOS, as - // shown in issue #101653 and the discussion in PR #101792. - // - // It could be required in some cases of cross-compiling with - // `-Zgcc-ld=lld`, but this is generally unspecified, and we don't know - // which specific versions of clang, macOS SDK, host and target OS - // combinations impact us here. - // - // So we do a simple first-approximation until we know more of what the - // Apple targets require (and which would be handled prior to hitting this - // `-Zgcc-ld=lld` codepath anyway), but the expectation is that until then - // this should be manually passed if needed. We specify the target when - // targeting a different linker flavor on macOS, and that's also always - // the case when targeting WASM. - if sess.target.linker_flavor != sess.host.linker_flavor { - cmd.arg(format!("--target={}", sess.target.llvm_target)); - } - } - } - } - } else { - sess.emit_fatal(errors::OptionGccOnly); +/// When using the linker flavors opting in to `lld`, or the unstable `-Zgcc-ld=lld` flag, add the +/// necessary paths and arguments to invoke it: +/// - when the self-contained linker flag is active: the build of `lld` distributed with rustc, +/// - or any `lld` available to `cc`. +fn add_lld_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { + let unstable_use_lld = sess.opts.unstable_opts.gcc_ld.is_some(); + debug!("add_lld_args requested, flavor: '{flavor:?}', `-Zgcc-ld=lld`: {unstable_use_lld}"); + + // Sanity check: using the old unstable `-Zgcc-ld=lld` option requires a `cc`-using flavor. + let flavor_uses_cc = flavor.uses_cc(); + if unstable_use_lld && !flavor_uses_cc { + sess.emit_fatal(errors::OptionGccOnly); + } + + // If the flavor doesn't use a C/C++ compiler to invoke the linker, or doesn't opt in to `lld`, + // we don't need to do anything. + let use_lld = flavor.uses_lld() || unstable_use_lld; + if !flavor_uses_cc || !use_lld { + return; + } + + let self_contained_linker = sess.opts.cg.link_self_contained.linker(); + + // FIXME: some targets default to using `lld`, but users can only override the linker on the CLI + // and cannot yet select the precise linker flavor to opt out of that. See for example issue + // #113597 for the `thumbv6m-none-eabi` target: a driver is used, and its default linker + // conflicts with the target's flavor, causing unexpected arguments being passed. + // + // Until the new `LinkerFlavor`-like CLI options are stabilized, we only adopt MCP510's behavior + // if its dedicated unstable CLI flags are used, to keep the current sub-optimal stable + // behavior. + let using_mcp510 = + self_contained_linker || sess.opts.cg.linker_flavor.is_some_and(|f| f.is_unstable()); + if !using_mcp510 && !unstable_use_lld { + return; + } + + // 1. Implement the "self-contained" part of this feature by adding rustc distribution + // directories to the tool's search path. + if self_contained_linker || unstable_use_lld { + for path in sess.get_tools_search_paths(false) { + cmd.arg({ + let mut arg = OsString::from("-B"); + arg.push(path.join("gcc-ld")); + arg + }); + } + } + + // 2. Implement the "linker flavor" part of this feature by asking `cc` to use some kind of + // `lld` as the linker. + cmd.arg("-fuse-ld=lld"); + + if !flavor.is_gnu() { + // Tell clang to use a non-default LLD flavor. + // Gcc doesn't understand the target option, but we currently assume + // that gcc is not used for Apple and Wasm targets (#97402). + // + // Note that we don't want to do that by default on macOS: e.g. passing a + // 10.7 target to LLVM works, but not to recent versions of clang/macOS, as + // shown in issue #101653 and the discussion in PR #101792. + // + // It could be required in some cases of cross-compiling with + // `-Zgcc-ld=lld`, but this is generally unspecified, and we don't know + // which specific versions of clang, macOS SDK, host and target OS + // combinations impact us here. + // + // So we do a simple first-approximation until we know more of what the + // Apple targets require (and which would be handled prior to hitting this + // `-Zgcc-ld=lld` codepath anyway), but the expectation is that until then + // this should be manually passed if needed. We specify the target when + // targeting a different linker flavor on macOS, and that's also always + // the case when targeting WASM. + if sess.target.linker_flavor != sess.host.linker_flavor { + cmd.arg(format!("--target={}", sess.target.llvm_target)); } } } diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index cd56f85cc..8ac86fa4b 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -13,6 +13,7 @@ use std::{env, mem, str}; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_metadata::find_native_static_library; use rustc_middle::middle::dependency_format::Linkage; +use rustc_middle::middle::exported_symbols; use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind}; use rustc_middle::ty::TyCtxt; use rustc_session::config::{self, CrateType, DebugInfo, LinkerPluginLto, Lto, OptLevel, Strip}; @@ -659,8 +660,6 @@ impl<'a> Linker for GccLinker<'a> { return; } - // FIXME(#99978) hide #[no_mangle] symbols for proc-macros - let is_windows = self.sess.target.is_like_windows; let path = tmpdir.join(if is_windows { "list.def" } else { "list" }); @@ -1679,8 +1678,15 @@ pub(crate) fn exported_symbols(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec<St return exports.iter().map(ToString::to_string).collect(); } - let mut symbols = Vec::new(); + if let CrateType::ProcMacro = crate_type { + exported_symbols_for_proc_macro_crate(tcx) + } else { + exported_symbols_for_non_proc_macro(tcx, crate_type) + } +} +fn exported_symbols_for_non_proc_macro(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec<String> { + let mut symbols = Vec::new(); let export_threshold = symbol_export::crates_export_threshold(&[crate_type]); for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| { if info.level.is_below_threshold(export_threshold) { @@ -1691,6 +1697,19 @@ pub(crate) fn exported_symbols(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec<St symbols } +fn exported_symbols_for_proc_macro_crate(tcx: TyCtxt<'_>) -> Vec<String> { + // `exported_symbols` will be empty when !should_codegen. + if !tcx.sess.opts.output_types.should_codegen() { + return Vec::new(); + } + + let stable_crate_id = tcx.sess.local_stable_crate_id(); + let proc_macro_decls_name = tcx.sess.generate_proc_macro_decls_symbol(stable_crate_id); + let metadata_symbol_name = exported_symbols::metadata_symbol_name(tcx); + + vec![proc_macro_decls_name, metadata_symbol_name] +} + pub(crate) fn linked_symbols( tcx: TyCtxt<'_>, crate_type: CrateType, diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index ad27b854d..00e6acb5c 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -6,8 +6,8 @@ use std::path::Path; use object::write::{self, StandardSegment, Symbol, SymbolSection}; use object::{ - elf, pe, Architecture, BinaryFormat, Endianness, FileFlags, Object, ObjectSection, - SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope, + elf, pe, xcoff, Architecture, BinaryFormat, Endianness, FileFlags, Object, ObjectSection, + ObjectSymbol, SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope, }; use snap::write::FrameEncoder; @@ -20,7 +20,7 @@ use rustc_metadata::EncodedMetadata; use rustc_session::cstore::MetadataLoader; use rustc_session::Session; use rustc_target::abi::Endian; -use rustc_target::spec::{RelocModel, Target}; +use rustc_target::spec::{ef_avr_arch, RelocModel, Target}; /// The default metadata loader. This is used by cg_llvm and cg_clif. /// @@ -35,6 +35,8 @@ use rustc_target::spec::{RelocModel, Target}; #[derive(Debug)] pub struct DefaultMetadataLoader; +static AIX_METADATA_SYMBOL_NAME: &'static str = "__aix_rust_metadata"; + fn load_metadata_with( path: &Path, f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>, @@ -48,7 +50,7 @@ fn load_metadata_with( } impl MetadataLoader for DefaultMetadataLoader { - fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<OwnedSlice, String> { + fn get_rlib_metadata(&self, target: &Target, path: &Path) -> Result<OwnedSlice, String> { load_metadata_with(path, |data| { let archive = object::read::archive::ArchiveFile::parse(&*data) .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; @@ -60,7 +62,11 @@ impl MetadataLoader for DefaultMetadataLoader { let data = entry .data(data) .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; - return search_for_section(path, data, ".rmeta"); + if target.is_like_aix { + return get_metadata_xcoff(path, data); + } else { + return search_for_section(path, data, ".rmeta"); + } } } @@ -68,8 +74,12 @@ impl MetadataLoader for DefaultMetadataLoader { }) } - fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<OwnedSlice, String> { - load_metadata_with(path, |data| search_for_section(path, data, ".rustc")) + fn get_dylib_metadata(&self, target: &Target, path: &Path) -> Result<OwnedSlice, String> { + if target.is_like_aix { + load_metadata_with(path, |data| get_metadata_xcoff(path, data)) + } else { + load_metadata_with(path, |data| search_for_section(path, data, ".rustc")) + } } } @@ -141,6 +151,33 @@ fn add_gnu_property_note( file.append_section_data(section, &data, 8); } +pub(super) fn get_metadata_xcoff<'a>(path: &Path, data: &'a [u8]) -> Result<&'a [u8], String> { + let Ok(file) = object::File::parse(data) else { + return Ok(data); + }; + let info_data = search_for_section(path, data, ".info")?; + if let Some(metadata_symbol) = + file.symbols().find(|sym| sym.name() == Ok(AIX_METADATA_SYMBOL_NAME)) + { + let offset = metadata_symbol.address() as usize; + if offset < 4 { + return Err(format!("Invalid metadata symbol offset: {}", offset)); + } + // The offset specifies the location of rustc metadata in the comment section. + // The metadata is preceded by a 4-byte length field. + let len = u32::from_be_bytes(info_data[(offset - 4)..offset].try_into().unwrap()) as usize; + if offset + len > (info_data.len() as usize) { + return Err(format!( + "Metadata at offset {} with size {} is beyond .info section", + offset, len + )); + } + return Ok(&info_data[offset..(offset + len)]); + } else { + return Err(format!("Unable to find symbol {}", AIX_METADATA_SYMBOL_NAME)); + }; +} + pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static>> { let endianness = match sess.target.options.endian { Endian::Little => Endianness::Little, @@ -183,6 +220,8 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static BinaryFormat::MachO } else if sess.target.is_like_windows { BinaryFormat::Coff + } else if sess.target.is_like_aix { + BinaryFormat::Xcoff } else { BinaryFormat::Elf }; @@ -245,8 +284,24 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static e_flags } Architecture::LoongArch64 => { - // Source: https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html#_e_flags_identifies_abi_type_and_version - elf::EF_LARCH_OBJABI_V1 | elf::EF_LARCH_ABI_DOUBLE_FLOAT + // Source: https://github.com/loongson/la-abi-specs/blob/release/laelf.adoc#e_flags-identifies-abi-type-and-version + let mut e_flags: u32 = elf::EF_LARCH_OBJABI_V1; + let features = &sess.target.options.features; + + // Select the appropriate floating-point ABI + if features.contains("+d") { + e_flags |= elf::EF_LARCH_ABI_DOUBLE_FLOAT; + } else if features.contains("+f") { + e_flags |= elf::EF_LARCH_ABI_SINGLE_FLOAT; + } else { + e_flags |= elf::EF_LARCH_ABI_SOFT_FLOAT; + } + e_flags + } + Architecture::Avr => { + // Resolve the ISA revision and set + // the appropriate EF_AVR_ARCH flag. + ef_avr_arch(&sess.target.options.cpu) } _ => 0, }; @@ -351,11 +406,15 @@ pub fn create_wrapper_file( // to add a case above. return (data.to_vec(), MetadataPosition::Last); }; - let section = file.add_section( - file.segment_name(StandardSegment::Debug).to_vec(), - section_name, - SectionKind::Debug, - ); + let section = if file.format() == BinaryFormat::Xcoff { + file.add_section(Vec::new(), b".info".to_vec(), SectionKind::Debug) + } else { + file.add_section( + file.segment_name(StandardSegment::Debug).to_vec(), + section_name, + SectionKind::Debug, + ) + }; match file.format() { BinaryFormat::Coff => { file.section_mut(section).flags = @@ -365,6 +424,31 @@ pub fn create_wrapper_file( file.section_mut(section).flags = SectionFlags::Elf { sh_flags: elf::SHF_EXCLUDE as u64 }; } + BinaryFormat::Xcoff => { + // AIX system linker may aborts if it meets a valid XCOFF file in archive with no .text, no .data and no .bss. + file.add_section(Vec::new(), b".text".to_vec(), SectionKind::Text); + file.section_mut(section).flags = + SectionFlags::Xcoff { s_flags: xcoff::STYP_INFO as u32 }; + + let len = data.len() as u32; + let offset = file.append_section_data(section, &len.to_be_bytes(), 1); + // Add a symbol referring to the data in .info section. + file.add_symbol(Symbol { + name: AIX_METADATA_SYMBOL_NAME.into(), + value: offset + 4, + size: 0, + kind: SymbolKind::Unknown, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::Section(section), + flags: SymbolFlags::Xcoff { + n_sclass: xcoff::C_INFO, + x_smtyp: xcoff::C_HIDEXT, + x_smclas: xcoff::C_HIDEXT, + containing_csect: None, + }, + }); + } _ => {} }; file.append_section_data(section, data, 1); @@ -401,6 +485,9 @@ pub fn create_compressed_metadata_file( let Some(mut file) = create_object_file(sess) else { return compressed.to_vec(); }; + if file.format() == BinaryFormat::Xcoff { + return create_compressed_metadata_file_for_xcoff(file, &compressed, symbol_name); + } let section = file.add_section( file.segment_name(StandardSegment::Data).to_vec(), b".rustc".to_vec(), @@ -430,3 +517,61 @@ pub fn create_compressed_metadata_file( file.write().unwrap() } + +/// * Xcoff - On AIX, custom sections are merged into predefined sections, +/// so custom .rustc section is not preserved during linking. +/// For this reason, we store metadata in predefined .info section, and +/// define a symbol to reference the metadata. To preserve metadata during +/// linking on AIX, we have to +/// 1. Create an empty .text section, a empty .data section. +/// 2. Define an empty symbol named `symbol_name` inside .data section. +/// 3. Define an symbol named `AIX_METADATA_SYMBOL_NAME` referencing +/// data inside .info section. +/// From XCOFF's view, (2) creates a csect entry in the symbol table, the +/// symbol created by (3) is a info symbol for the preceding csect. Thus +/// two symbols are preserved during linking and we can use the second symbol +/// to reference the metadata. +pub fn create_compressed_metadata_file_for_xcoff( + mut file: write::Object<'_>, + data: &[u8], + symbol_name: &str, +) -> Vec<u8> { + assert!(file.format() == BinaryFormat::Xcoff); + // AIX system linker may aborts if it meets a valid XCOFF file in archive with no .text, no .data and no .bss. + file.add_section(Vec::new(), b".text".to_vec(), SectionKind::Text); + let data_section = file.add_section(Vec::new(), b".data".to_vec(), SectionKind::Data); + let section = file.add_section(Vec::new(), b".info".to_vec(), SectionKind::Debug); + file.add_file_symbol("lib.rmeta".into()); + file.section_mut(section).flags = SectionFlags::Xcoff { s_flags: xcoff::STYP_INFO as u32 }; + // Add a global symbol to data_section. + file.add_symbol(Symbol { + name: symbol_name.as_bytes().into(), + value: 0, + size: 0, + kind: SymbolKind::Data, + scope: SymbolScope::Dynamic, + weak: true, + section: SymbolSection::Section(data_section), + flags: SymbolFlags::None, + }); + let len = data.len() as u32; + let offset = file.append_section_data(section, &len.to_be_bytes(), 1); + // Add a symbol referring to the rustc metadata. + file.add_symbol(Symbol { + name: AIX_METADATA_SYMBOL_NAME.into(), + value: offset + 4, // The metadata is preceded by a 4-byte length field. + size: 0, + kind: SymbolKind::Unknown, + scope: SymbolScope::Dynamic, + weak: false, + section: SymbolSection::Section(section), + flags: SymbolFlags::Xcoff { + n_sclass: xcoff::C_INFO, + x_smtyp: xcoff::C_HIDEXT, + x_smclas: xcoff::C_HIDEXT, + containing_csect: None, + }, + }); + file.append_section_data(section, data, 1); + file.write().unwrap() +} diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index c323372bd..ececa29b2 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -9,11 +9,9 @@ use crate::{ }; use jobserver::{Acquired, Client}; use rustc_ast::attr; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_data_structures::memmap::Mmap; -use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_data_structures::profiling::TimingGuard; -use rustc_data_structures::profiling::VerboseTimingGuard; +use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard}; use rustc_data_structures::sync::Lrc; use rustc_errors::emitter::Emitter; use rustc_errors::{translation::Translate, DiagnosticId, FatalError, Handler, Level}; @@ -23,12 +21,13 @@ use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_incremental::{ copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess, }; +use rustc_metadata::fs::copy_to_stdout; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::middle::exported_symbols::SymbolExportInfo; use rustc_middle::ty::TyCtxt; use rustc_session::cgu_reuse_tracker::CguReuseTracker; -use rustc_session::config::{self, CrateType, Lto, OutputFilenames, OutputType}; +use rustc_session::config::{self, CrateType, Lto, OutFileName, OutputFilenames, OutputType}; use rustc_session::config::{Passes, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::source_map::SourceMap; @@ -36,6 +35,7 @@ use rustc_span::symbol::sym; use rustc_span::{BytePos, FileName, InnerSpan, Pos, Span}; use rustc_target::spec::{MergeFunctions, SanitizerSet}; +use crate::errors::ErrorCreatingRemarkDir; use std::any::Any; use std::borrow::Cow; use std::fs; @@ -321,7 +321,6 @@ pub type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo #[derive(Clone)] pub struct CodegenContext<B: WriteBackendMethods> { // Resources needed when running LTO - pub backend: B, pub prof: SelfProfilerRef, pub lto: Lto, pub save_temps: bool, @@ -339,18 +338,17 @@ pub struct CodegenContext<B: WriteBackendMethods> { pub msvc_imps_needed: bool, pub is_pe_coff: bool, pub target_can_use_split_dwarf: bool, - pub target_pointer_width: u32, pub target_arch: String, - pub debuginfo: config::DebugInfo, pub split_debuginfo: rustc_target::spec::SplitDebuginfo, pub split_dwarf_kind: rustc_session::config::SplitDwarfKind, - /// Number of cgus excluding the allocator/metadata modules - pub total_cgus: usize, /// Handler to use for diagnostics produced during codegen. pub diag_emitter: SharedEmitter, /// LLVM optimizations for which we want to print remarks. pub remark: Passes, + /// Directory into which should the LLVM optimization remarks be written. + /// If `None`, they will be written to stderr. + pub remark_dir: Option<PathBuf>, /// Worker thread number pub worker: usize, /// The incremental compilation session directory, or None if we are not @@ -442,7 +440,6 @@ pub fn start_async_codegen<B: ExtraBackendMethods>( target_cpu: String, metadata: EncodedMetadata, metadata_module: Option<CompiledModule>, - total_cgus: usize, ) -> OngoingCodegen<B> { let (coordinator_send, coordinator_receive) = channel(); let sess = tcx.sess; @@ -470,7 +467,6 @@ pub fn start_async_codegen<B: ExtraBackendMethods>( shared_emitter, codegen_worker_send, coordinator_receive, - total_cgus, sess.jobserver.clone(), Arc::new(regular_config), Arc::new(metadata_config), @@ -498,8 +494,8 @@ pub fn start_async_codegen<B: ExtraBackendMethods>( fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( sess: &Session, compiled_modules: &CompiledModules, -) -> FxHashMap<WorkProductId, WorkProduct> { - let mut work_products = FxHashMap::default(); +) -> FxIndexMap<WorkProductId, WorkProduct> { + let mut work_products = FxIndexMap::default(); if sess.opts.incremental.is_none() { return work_products; @@ -535,9 +531,16 @@ fn produce_final_output_artifacts( let mut user_wants_objects = false; // Produce final compile outputs. - let copy_gracefully = |from: &Path, to: &Path| { - if let Err(e) = fs::copy(from, to) { - sess.emit_err(errors::CopyPath::new(from, to, e)); + let copy_gracefully = |from: &Path, to: &OutFileName| match to { + OutFileName::Stdout => { + if let Err(e) = copy_to_stdout(from) { + sess.emit_err(errors::CopyPath::new(from, to.as_path(), e)); + } + } + OutFileName::Real(path) => { + if let Err(e) = fs::copy(from, path) { + sess.emit_err(errors::CopyPath::new(from, path, e)); + } } }; @@ -547,7 +550,12 @@ fn produce_final_output_artifacts( // to copy `foo.0.x` to `foo.x`. let module_name = Some(&compiled_modules.modules[0].name[..]); let path = crate_output.temp_path(output_type, module_name); - copy_gracefully(&path, &crate_output.path(output_type)); + let output = crate_output.path(output_type); + if !output_type.is_text_output() && output.is_tty() { + sess.emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() }); + } else { + copy_gracefully(&path, &output); + } if !sess.opts.cg.save_temps && !keep_numbered { // The user just wants `foo.x`, not `foo.#module-name#.x`. ensure_removed(sess.diagnostic(), &path); @@ -633,10 +641,10 @@ fn produce_final_output_artifacts( // rlib. let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe); - let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units() > 1; + let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1; let keep_numbered_objects = - needs_crate_object || (user_wants_objects && sess.codegen_units() > 1); + needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1); for module in compiled_modules.modules.iter() { if let Some(ref path) = module.object { @@ -674,7 +682,7 @@ fn produce_final_output_artifacts( // These are used in linking steps and will be cleaned up afterward. } -pub enum WorkItem<B: WriteBackendMethods> { +pub(crate) enum WorkItem<B: WriteBackendMethods> { /// Optimize a newly codegened, totally unoptimized module. Optimize(ModuleCodegen<B::Module>), /// Copy the post-LTO artifacts from the incremental cache to the output @@ -692,49 +700,57 @@ impl<B: WriteBackendMethods> WorkItem<B> { } } - fn start_profiling<'a>(&self, cgcx: &'a CodegenContext<B>) -> TimingGuard<'a> { - match *self { - WorkItem::Optimize(ref m) => { - cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name) - } - WorkItem::CopyPostLtoArtifacts(ref m) => cgcx - .prof - .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &*m.name), - WorkItem::LTO(ref m) => { - cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()) - } - } - } - /// Generate a short description of this work item suitable for use as a thread name. fn short_description(&self) -> String { - // `pthread_setname()` on *nix is limited to 15 characters and longer names are ignored. - // Use very short descriptions in this case to maximize the space available for the module name. - // Windows does not have that limitation so use slightly more descriptive names there. + // `pthread_setname()` on *nix ignores anything beyond the first 15 + // bytes. Use short descriptions to maximize the space available for + // the module name. + #[cfg(not(windows))] + fn desc(short: &str, _long: &str, name: &str) -> String { + // The short label is three bytes, and is followed by a space. That + // leaves 11 bytes for the CGU name. How we obtain those 11 bytes + // depends on the the CGU name form. + // + // - Non-incremental, e.g. `regex.f10ba03eb5ec7975-cgu.0`: the part + // before the `-cgu.0` is the same for every CGU, so use the + // `cgu.0` part. The number suffix will be different for each + // CGU. + // + // - Incremental (normal), e.g. `2i52vvl2hco29us0`: use the whole + // name because each CGU will have a unique ASCII hash, and the + // first 11 bytes will be enough to identify it. + // + // - Incremental (with `-Zhuman-readable-cgu-names`), e.g. + // `regex.f10ba03eb5ec7975-re_builder.volatile`: use the whole + // name. The first 11 bytes won't be enough to uniquely identify + // it, but no obvious substring will, and this is a rarely used + // option so it doesn't matter much. + // + assert_eq!(short.len(), 3); + let name = if let Some(index) = name.find("-cgu.") { + &name[index + 1..] // +1 skips the leading '-'. + } else { + name + }; + format!("{short} {name}") + } + + // Windows has no thread name length limit, so use more descriptive names. + #[cfg(windows)] + fn desc(_short: &str, long: &str, name: &str) -> String { + format!("{long} {name}") + } + match self { - WorkItem::Optimize(m) => { - #[cfg(windows)] - return format!("optimize module {}", m.name); - #[cfg(not(windows))] - return format!("opt {}", m.name); - } - WorkItem::CopyPostLtoArtifacts(m) => { - #[cfg(windows)] - return format!("copy LTO artifacts for {}", m.name); - #[cfg(not(windows))] - return format!("copy {}", m.name); - } - WorkItem::LTO(m) => { - #[cfg(windows)] - return format!("LTO module {}", m.name()); - #[cfg(not(windows))] - return format!("LTO {}", m.name()); - } + WorkItem::Optimize(m) => desc("opt", "optimize module {}", &m.name), + WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for {}", &m.name), + WorkItem::LTO(m) => desc("lto", "LTO module {}", m.name()), } } } -enum WorkItemResult<B: WriteBackendMethods> { +/// A result produced by the backend. +pub(crate) enum WorkItemResult<B: WriteBackendMethods> { Compiled(CompiledModule), NeedsLink(ModuleCodegen<B::Module>), NeedsFatLTO(FatLTOInput<B>), @@ -746,21 +762,6 @@ pub enum FatLTOInput<B: WriteBackendMethods> { InMemory(ModuleCodegen<B::Module>), } -fn execute_work_item<B: ExtraBackendMethods>( - cgcx: &CodegenContext<B>, - work_item: WorkItem<B>, -) -> Result<WorkItemResult<B>, FatalError> { - let module_config = cgcx.config(work_item.module_kind()); - - match work_item { - WorkItem::Optimize(module) => execute_optimize_work_item(cgcx, module, module_config), - WorkItem::CopyPostLtoArtifacts(module) => { - Ok(execute_copy_from_cache_work_item(cgcx, module, module_config)) - } - WorkItem::LTO(module) => execute_lto_work_item(cgcx, module, module_config), - } -} - /// Actual LTO type we end up choosing based on multiple factors. pub enum ComputedLtoType { No, @@ -941,38 +942,41 @@ fn finish_intra_module_work<B: ExtraBackendMethods>( } } -pub enum Message<B: WriteBackendMethods> { +/// Messages sent to the coordinator. +pub(crate) enum Message<B: WriteBackendMethods> { + /// A jobserver token has become available. Sent from the jobserver helper + /// thread. Token(io::Result<Acquired>), - NeedsFatLTO { - result: FatLTOInput<B>, - worker_id: usize, - }, - NeedsThinLTO { - name: String, - thin_buffer: B::ThinBuffer, - worker_id: usize, - }, - NeedsLink { - module: ModuleCodegen<B::Module>, - worker_id: usize, - }, - Done { - result: Result<CompiledModule, Option<WorkerFatalError>>, - worker_id: usize, - }, - CodegenDone { - llvm_work_item: WorkItem<B>, - cost: u64, - }, + + /// The backend has finished processing a work item for a codegen unit. + /// Sent from a backend worker thread. + WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize }, + + /// The frontend has finished generating something (backend IR or a + /// post-LTO artifact) for a codegen unit, and it should be passed to the + /// backend. Sent from the main thread. + CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 }, + + /// Similar to `CodegenDone`, but for reusing a pre-LTO artifact + /// Sent from the main thread. AddImportOnlyModule { module_data: SerializedModule<B::ModuleBuffer>, work_product: WorkProduct, }, + + /// The frontend has finished generating everything for all codegen units. + /// Sent from the main thread. CodegenComplete, - CodegenItem, + + /// Some normal-ish compiler error occurred, and codegen should be wound + /// down. Sent from the main thread. CodegenAborted, } +/// A message sent from the coordinator thread to the main thread telling it to +/// process another codegen unit. +pub struct CguMessage; + type DiagnosticArgName<'source> = Cow<'source, str>; struct Diagnostic { @@ -994,9 +998,8 @@ fn start_executing_work<B: ExtraBackendMethods>( tcx: TyCtxt<'_>, crate_info: &CrateInfo, shared_emitter: SharedEmitter, - codegen_worker_send: Sender<Message<B>>, + codegen_worker_send: Sender<CguMessage>, coordinator_receive: Receiver<Box<dyn Any + Send>>, - total_cgus: usize, jobserver: Client, regular_config: Arc<ModuleConfig>, metadata_config: Arc<ModuleConfig>, @@ -1063,8 +1066,18 @@ fn start_executing_work<B: ExtraBackendMethods>( tcx.backend_optimization_level(()) }; let backend_features = tcx.global_backend_features(()); + + let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir { + let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize()); + match result { + Ok(dir) => Some(dir), + Err(error) => sess.emit_fatal(ErrorCreatingRemarkDir { error }), + } + } else { + None + }; + let cgcx = CodegenContext::<B> { - backend: backend.clone(), crate_types: sess.crate_types().to_vec(), each_linked_rlib_for_lto, lto: sess.lto(), @@ -1075,6 +1088,7 @@ fn start_executing_work<B: ExtraBackendMethods>( prof: sess.prof.clone(), exported_symbols, remark: sess.opts.cg.remark.clone(), + remark_dir, worker: 0, incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()), cgu_reuse_tracker: sess.cgu_reuse_tracker.clone(), @@ -1085,13 +1099,10 @@ fn start_executing_work<B: ExtraBackendMethods>( metadata_module_config: metadata_config, allocator_module_config: allocator_config, tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features), - total_cgus, msvc_imps_needed: msvc_imps_needed(tcx), is_pe_coff: tcx.sess.target.is_like_windows, target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(), - target_pointer_width: tcx.sess.target.pointer_width, target_arch: tcx.sess.target.arch.to_string(), - debuginfo: tcx.sess.opts.debuginfo, split_debuginfo: tcx.sess.split_debuginfo(), split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind, }; @@ -1253,10 +1264,19 @@ fn start_executing_work<B: ExtraBackendMethods>( let mut needs_thin_lto = Vec::new(); let mut lto_import_only_modules = Vec::new(); let mut started_lto = false; - let mut codegen_aborted = false; - // This flag tracks whether all items have gone through codegens - let mut codegen_done = false; + /// Possible state transitions: + /// - Ongoing -> Completed + /// - Ongoing -> Aborted + /// - Completed -> Aborted + #[derive(Debug, PartialEq)] + enum CodegenState { + Ongoing, + Completed, + Aborted, + } + use CodegenState::*; + let mut codegen_state = Ongoing; // This is the queue of LLVM work items that still need processing. let mut work_items = Vec::<(WorkItem<B>, u64)>::new(); @@ -1276,10 +1296,10 @@ fn start_executing_work<B: ExtraBackendMethods>( // wait for all existing work to finish, so many of the conditions here // only apply if codegen hasn't been aborted as they represent pending // work to be done. - while !codegen_done + while codegen_state == Ongoing || running > 0 || main_thread_worker_state == MainThreadWorkerState::LLVMing - || (!codegen_aborted + || (codegen_state == Completed && !(work_items.is_empty() && needs_fat_lto.is_empty() && needs_thin_lto.is_empty() @@ -1289,7 +1309,7 @@ fn start_executing_work<B: ExtraBackendMethods>( // While there are still CGUs to be codegened, the coordinator has // to decide how to utilize the compiler processes implicit Token: // For codegenning more CGU or for running them through LLVM. - if !codegen_done { + if codegen_state == Ongoing { if main_thread_worker_state == MainThreadWorkerState::Idle { // Compute the number of workers that will be running once we've taken as many // items from the work queue as we can, plus one for the main thread. It's not @@ -1302,9 +1322,9 @@ fn start_executing_work<B: ExtraBackendMethods>( let anticipated_running = running + additional_running + 1; if !queue_full_enough(work_items.len(), anticipated_running) { - // The queue is not full enough, codegen more items: - if codegen_worker_send.send(Message::CodegenItem).is_err() { - panic!("Could not send Message::CodegenItem to main thread") + // The queue is not full enough, process more codegen units: + if codegen_worker_send.send(CguMessage).is_err() { + panic!("Could not send CguMessage to main thread") } main_thread_worker_state = MainThreadWorkerState::Codegenning; } else { @@ -1326,10 +1346,7 @@ fn start_executing_work<B: ExtraBackendMethods>( spawn_work(cgcx, item); } } - } else if codegen_aborted { - // don't queue up any more work if codegen was aborted, we're - // just waiting for our existing children to finish - } else { + } else if codegen_state == Completed { // If we've finished everything related to normal codegen // then it must be the case that we've got some LTO work to do. // Perform the serial work here of figuring out what we're @@ -1396,11 +1413,15 @@ fn start_executing_work<B: ExtraBackendMethods>( // Already making good use of that token } } + } else { + // Don't queue up any more work if codegen was aborted, we're + // just waiting for our existing children to finish. + assert!(codegen_state == Aborted); } // Spin up what work we can, only doing this while we've got available // parallelism slots and work left to spawn. - while !codegen_aborted && !work_items.is_empty() && running < tokens.len() { + while codegen_state != Aborted && !work_items.is_empty() && running < tokens.len() { let (item, _) = work_items.pop().unwrap(); maybe_start_llvm_timer(prof, cgcx.config(item.module_kind()), &mut llvm_start_time); @@ -1452,8 +1473,7 @@ fn start_executing_work<B: ExtraBackendMethods>( Err(e) => { let msg = &format!("failed to acquire jobserver token: {}", e); shared_emitter.fatal(msg); - codegen_done = true; - codegen_aborted = true; + codegen_state = Aborted; } } } @@ -1481,7 +1501,9 @@ fn start_executing_work<B: ExtraBackendMethods>( } Message::CodegenComplete => { - codegen_done = true; + if codegen_state != Aborted { + codegen_state = Completed; + } assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); main_thread_worker_state = MainThreadWorkerState::Idle; } @@ -1493,58 +1515,59 @@ fn start_executing_work<B: ExtraBackendMethods>( // then conditions above will ensure no more work is spawned but // we'll keep executing this loop until `running` hits 0. Message::CodegenAborted => { - codegen_done = true; - codegen_aborted = true; + codegen_state = Aborted; } - Message::Done { result: Ok(compiled_module), worker_id } => { + + Message::WorkItem { result, worker_id } => { free_worker(worker_id); - match compiled_module.kind { - ModuleKind::Regular => { - compiled_modules.push(compiled_module); + + match result { + Ok(WorkItemResult::Compiled(compiled_module)) => { + match compiled_module.kind { + ModuleKind::Regular => { + compiled_modules.push(compiled_module); + } + ModuleKind::Allocator => { + assert!(compiled_allocator_module.is_none()); + compiled_allocator_module = Some(compiled_module); + } + ModuleKind::Metadata => bug!("Should be handled separately"), + } + } + Ok(WorkItemResult::NeedsLink(module)) => { + needs_link.push(module); + } + Ok(WorkItemResult::NeedsFatLTO(fat_lto_input)) => { + assert!(!started_lto); + needs_fat_lto.push(fat_lto_input); + } + Ok(WorkItemResult::NeedsThinLTO(name, thin_buffer)) => { + assert!(!started_lto); + needs_thin_lto.push((name, thin_buffer)); + } + Err(Some(WorkerFatalError)) => { + // Like `CodegenAborted`, wait for remaining work to finish. + codegen_state = Aborted; } - ModuleKind::Allocator => { - assert!(compiled_allocator_module.is_none()); - compiled_allocator_module = Some(compiled_module); + Err(None) => { + // If the thread failed that means it panicked, so + // we abort immediately. + bug!("worker thread panicked"); } - ModuleKind::Metadata => bug!("Should be handled separately"), } } - Message::NeedsLink { module, worker_id } => { - free_worker(worker_id); - needs_link.push(module); - } - Message::NeedsFatLTO { result, worker_id } => { - assert!(!started_lto); - free_worker(worker_id); - needs_fat_lto.push(result); - } - Message::NeedsThinLTO { name, thin_buffer, worker_id } => { - assert!(!started_lto); - free_worker(worker_id); - needs_thin_lto.push((name, thin_buffer)); - } + Message::AddImportOnlyModule { module_data, work_product } => { assert!(!started_lto); - assert!(!codegen_done); + assert_eq!(codegen_state, Ongoing); assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); lto_import_only_modules.push((module_data, work_product)); main_thread_worker_state = MainThreadWorkerState::Idle; } - // If the thread failed that means it panicked, so we abort immediately. - Message::Done { result: Err(None), worker_id: _ } => { - bug!("worker thread panicked"); - } - Message::Done { result: Err(Some(WorkerFatalError)), worker_id } => { - // Similar to CodegenAborted, wait for remaining work to finish. - free_worker(worker_id); - codegen_done = true; - codegen_aborted = true; - } - Message::CodegenItem => bug!("the coordinator should not receive codegen requests"), } } - if codegen_aborted { + if codegen_state == Aborted { return Err(()); } @@ -1659,22 +1682,11 @@ fn spawn_work<B: ExtraBackendMethods>(cgcx: CodegenContext<B>, work: WorkItem<B> fn drop(&mut self) { let worker_id = self.worker_id; let msg = match self.result.take() { - Some(Ok(WorkItemResult::Compiled(m))) => { - Message::Done::<B> { result: Ok(m), worker_id } - } - Some(Ok(WorkItemResult::NeedsLink(m))) => { - Message::NeedsLink::<B> { module: m, worker_id } - } - Some(Ok(WorkItemResult::NeedsFatLTO(m))) => { - Message::NeedsFatLTO::<B> { result: m, worker_id } - } - Some(Ok(WorkItemResult::NeedsThinLTO(name, thin_buffer))) => { - Message::NeedsThinLTO::<B> { name, thin_buffer, worker_id } - } + Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id }, Some(Err(FatalError)) => { - Message::Done::<B> { result: Err(Some(WorkerFatalError)), worker_id } + Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id } } - None => Message::Done::<B> { result: Err(None), worker_id }, + None => Message::WorkItem::<B> { result: Err(None), worker_id }, }; drop(self.coordinator_send.send(Box::new(msg))); } @@ -1693,8 +1705,27 @@ fn spawn_work<B: ExtraBackendMethods>(cgcx: CodegenContext<B>, work: WorkItem<B> // as a diagnostic was already sent off to the main thread - just // surface that there was an error in this worker. bomb.result = { - let _prof_timer = work.start_profiling(&cgcx); - Some(execute_work_item(&cgcx, work)) + let module_config = cgcx.config(work.module_kind()); + + Some(match work { + WorkItem::Optimize(m) => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name); + execute_optimize_work_item(&cgcx, m, module_config) + } + WorkItem::CopyPostLtoArtifacts(m) => { + let _timer = cgcx.prof.generic_activity_with_arg( + "codegen_copy_artifacts_from_incr_cache", + &*m.name, + ); + Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config)) + } + WorkItem::LTO(m) => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()); + execute_lto_work_item(&cgcx, m, module_config) + } + }) }; }) .expect("failed to spawn thread"); @@ -1800,7 +1831,7 @@ impl SharedEmitterMain { handler.emit_diagnostic(&mut d); } Ok(SharedEmitterMessage::InlineAsmError(cookie, msg, level, source)) => { - let msg = msg.strip_prefix("error: ").unwrap_or(&msg); + let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string(); let mut err = match level { Level::Error { lint: false } => sess.struct_err(msg).forget_guarantee(), @@ -1878,14 +1909,14 @@ pub struct OngoingCodegen<B: ExtraBackendMethods> { pub metadata: EncodedMetadata, pub metadata_module: Option<CompiledModule>, pub crate_info: CrateInfo, - pub codegen_worker_receive: Receiver<Message<B>>, + pub codegen_worker_receive: Receiver<CguMessage>, pub shared_emitter_main: SharedEmitterMain, pub output_filenames: Arc<OutputFilenames>, pub coordinator: Coordinator<B>, } impl<B: ExtraBackendMethods> OngoingCodegen<B> { - pub fn join(self, sess: &Session) -> (CodegenResults, FxHashMap<WorkProductId, WorkProduct>) { + pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) { let _timer = sess.timer("finish_ongoing_codegen"); self.shared_emitter_main.check(sess, true); @@ -1910,7 +1941,7 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> { // FIXME: time_llvm_passes support - does this use a global context or // something? - if sess.codegen_units() == 1 && sess.opts.unstable_opts.time_llvm_passes { + if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes { self.backend.print_pass_timings() } @@ -1952,10 +1983,9 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> { pub fn wait_for_signal_to_codegen_item(&self) { match self.codegen_worker_receive.recv() { - Ok(Message::CodegenItem) => { - // Nothing to do + Ok(CguMessage) => { + // Ok to proceed. } - Ok(_) => panic!("unexpected message"), Err(_) => { // One of the LLVM threads must have panicked, fall through so // error handling can be reached. diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index ab7dd1ba8..9133601ec 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -199,7 +199,7 @@ fn vtable_ptr_ty<'tcx, Cx: CodegenMethods<'tcx>>( cx.scalar_pair_element_backend_type( cx.layout_of(match kind { // vtable is the second field of `*mut dyn Trait` - ty::Dyn => cx.tcx().mk_mut_ptr(target), + ty::Dyn => Ty::new_mut_ptr(cx.tcx(), target), // vtable is the second field of `dyn* Trait` ty::DynStar => target, }), @@ -295,7 +295,7 @@ pub fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let (base, info) = match bx.load_operand(src).val { OperandValue::Pair(base, info) => unsize_ptr(bx, base, src_ty, dst_ty, Some(info)), OperandValue::Immediate(base) => unsize_ptr(bx, base, src_ty, dst_ty, None), - OperandValue::Ref(..) => bug!(), + OperandValue::Ref(..) | OperandValue::ZeroSized => bug!(), }; OperandValue::Pair(base, info).store(bx, dst); } @@ -357,6 +357,13 @@ pub fn cast_shift_expr_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } +// Returns `true` if this session's target will use native wasm +// exceptions. This means that the VM does the unwinding for +// us +pub fn wants_wasm_eh(sess: &Session) -> bool { + sess.target.is_like_wasm && sess.target.os != "emscripten" +} + /// Returns `true` if this session's target will use SEH-based unwinding. /// /// This is only true for MSVC targets, and even then the 64-bit MSVC target @@ -366,6 +373,13 @@ pub fn wants_msvc_seh(sess: &Session) -> bool { sess.target.is_like_msvc } +/// Returns `true` if this session's target requires the new exception +/// handling LLVM IR instructions (catchpad / cleanuppad / ... instead +/// of landingpad) +pub fn wants_new_eh_instructions(sess: &Session) -> bool { + wants_wasm_eh(sess) || wants_msvc_seh(sess) +} + pub fn memcpy_ty<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, dst: Bx::Value, @@ -380,7 +394,19 @@ pub fn memcpy_ty<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( return; } - bx.memcpy(dst, dst_align, src, src_align, bx.cx().const_usize(size), flags); + if flags == MemFlags::empty() + && let Some(bty) = bx.cx().scalar_copy_backend_type(layout) + { + // I look forward to only supporting opaque pointers + let pty = bx.type_ptr_to(bty); + let src = bx.pointercast(src, pty); + let dst = bx.pointercast(dst, pty); + + let temp = bx.load(bty, src, src_align); + bx.store(temp, dst, dst_align); + } else { + bx.memcpy(dst, dst_align, src, src_align, bx.cx().const_usize(size), flags); + } } pub fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>( @@ -568,7 +594,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>( ) -> OngoingCodegen<B> { // Skip crate items and just output metadata in -Z no-codegen mode. if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, metadata, None, 1); + let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, metadata, None); ongoing_codegen.codegen_finished(tcx); @@ -619,14 +645,8 @@ pub fn codegen_crate<B: ExtraBackendMethods>( }) }); - let ongoing_codegen = start_async_codegen( - backend.clone(), - tcx, - target_cpu, - metadata, - metadata_module, - codegen_units.len(), - ); + let ongoing_codegen = + start_async_codegen(backend.clone(), tcx, target_cpu, metadata, metadata_module); // Codegen an allocator shim, if necessary. if let Some(kind) = allocator_kind_for_codegen(tcx) { diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs index e1abb73a5..5a6807599 100644 --- a/compiler/rustc_codegen_ssa/src/common.rs +++ b/compiler/rustc_codegen_ssa/src/common.rs @@ -132,7 +132,7 @@ pub fn build_langcall<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // all shifts). For 32- and 64-bit types, this matches the semantics // of Java. (See related discussion on #1877 and #10183.) -pub fn build_unchecked_lshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( +pub fn build_masked_lshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, lhs: Bx::Value, rhs: Bx::Value, @@ -143,7 +143,7 @@ pub fn build_unchecked_lshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx.shl(lhs, rhs) } -pub fn build_unchecked_rshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( +pub fn build_masked_rshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, lhs_t: Ty<'tcx>, lhs: Bx::Value, diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs deleted file mode 100644 index 1791ce4b3..000000000 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs +++ /dev/null @@ -1,89 +0,0 @@ -use rustc_middle::mir::coverage::{CounterValueReference, MappedExpressionIndex}; - -/// Must match the layout of `LLVMRustCounterKind`. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -pub enum CounterKind { - Zero = 0, - CounterValueReference = 1, - Expression = 2, -} - -/// A reference to an instance of an abstract "counter" that will yield a value in a coverage -/// report. Note that `id` has different interpretations, depending on the `kind`: -/// * For `CounterKind::Zero`, `id` is assumed to be `0` -/// * For `CounterKind::CounterValueReference`, `id` matches the `counter_id` of the injected -/// instrumentation counter (the `index` argument to the LLVM intrinsic -/// `instrprof.increment()`) -/// * For `CounterKind::Expression`, `id` is the index into the coverage map's array of -/// counter expressions. -/// -/// Corresponds to struct `llvm::coverage::Counter`. -/// -/// Must match the layout of `LLVMRustCounter`. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -pub struct Counter { - // Important: The layout (order and types of fields) must match its C++ counterpart. - pub kind: CounterKind, - id: u32, -} - -impl Counter { - /// Constructs a new `Counter` of kind `Zero`. For this `CounterKind`, the - /// `id` is not used. - pub fn zero() -> Self { - Self { kind: CounterKind::Zero, id: 0 } - } - - /// Constructs a new `Counter` of kind `CounterValueReference`, and converts - /// the given 1-based counter_id to the required 0-based equivalent for - /// the `Counter` encoding. - pub fn counter_value_reference(counter_id: CounterValueReference) -> Self { - Self { kind: CounterKind::CounterValueReference, id: counter_id.zero_based_index() } - } - - /// Constructs a new `Counter` of kind `Expression`. - pub fn expression(mapped_expression_index: MappedExpressionIndex) -> Self { - Self { kind: CounterKind::Expression, id: mapped_expression_index.into() } - } - - /// Returns true if the `Counter` kind is `Zero`. - pub fn is_zero(&self) -> bool { - matches!(self.kind, CounterKind::Zero) - } - - /// An explicitly-named function to get the ID value, making it more obvious - /// that the stored value is now 0-based. - pub fn zero_based_id(&self) -> u32 { - debug_assert!(!self.is_zero(), "`id` is undefined for CounterKind::Zero"); - self.id - } -} - -/// Corresponds to enum `llvm::coverage::CounterExpression::ExprKind`. -/// -/// Must match the layout of `LLVMRustCounterExprKind`. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -pub enum ExprKind { - Subtract = 0, - Add = 1, -} - -/// Corresponds to struct `llvm::coverage::CounterExpression`. -/// -/// Must match the layout of `LLVMRustCounterExpression`. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -pub struct CounterExpression { - pub kind: ExprKind, - pub lhs: Counter, - pub rhs: Counter, -} - -impl CounterExpression { - pub fn new(lhs: Counter, kind: ExprKind, rhs: Counter) -> Self { - Self { kind, lhs, rhs } - } -} diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs deleted file mode 100644 index e4da3b8de..000000000 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs +++ /dev/null @@ -1,347 +0,0 @@ -pub use super::ffi::*; - -use rustc_index::{IndexSlice, IndexVec}; -use rustc_middle::mir::coverage::{ - CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, - InjectedExpressionIndex, MappedExpressionIndex, Op, -}; -use rustc_middle::ty::Instance; -use rustc_middle::ty::TyCtxt; - -#[derive(Clone, Debug, PartialEq)] -pub struct Expression { - lhs: ExpressionOperandId, - op: Op, - rhs: ExpressionOperandId, - region: Option<CodeRegion>, -} - -/// Collects all of the coverage regions associated with (a) injected counters, (b) counter -/// expressions (additions or subtraction), and (c) unreachable regions (always counted as zero), -/// for a given Function. Counters and counter expressions have non-overlapping `id`s because they -/// can both be operands in an expression. This struct also stores the `function_source_hash`, -/// computed during instrumentation, and forwarded with counters. -/// -/// Note, it may be important to understand LLVM's definitions of `unreachable` regions versus "gap -/// regions" (or "gap areas"). A gap region is a code region within a counted region (either counter -/// or expression), but the line or lines in the gap region are not executable (such as lines with -/// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count -/// for a gap area is only used as the line execution count if there are no other regions on a -/// line." -#[derive(Debug)] -pub struct FunctionCoverage<'tcx> { - instance: Instance<'tcx>, - source_hash: u64, - is_used: bool, - counters: IndexVec<CounterValueReference, Option<CodeRegion>>, - expressions: IndexVec<InjectedExpressionIndex, Option<Expression>>, - unreachable_regions: Vec<CodeRegion>, -} - -impl<'tcx> FunctionCoverage<'tcx> { - /// Creates a new set of coverage data for a used (called) function. - pub fn new(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { - Self::create(tcx, instance, true) - } - - /// Creates a new set of coverage data for an unused (never called) function. - pub fn unused(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { - Self::create(tcx, instance, false) - } - - fn create(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, is_used: bool) -> Self { - let coverageinfo = tcx.coverageinfo(instance.def); - debug!( - "FunctionCoverage::create(instance={:?}) has coverageinfo={:?}. is_used={}", - instance, coverageinfo, is_used - ); - Self { - instance, - source_hash: 0, // will be set with the first `add_counter()` - is_used, - counters: IndexVec::from_elem_n(None, coverageinfo.num_counters as usize), - expressions: IndexVec::from_elem_n(None, coverageinfo.num_expressions as usize), - unreachable_regions: Vec::new(), - } - } - - /// Returns true for a used (called) function, and false for an unused function. - pub fn is_used(&self) -> bool { - self.is_used - } - - /// Sets the function source hash value. If called multiple times for the same function, all - /// calls should have the same hash value. - pub fn set_function_source_hash(&mut self, source_hash: u64) { - if self.source_hash == 0 { - self.source_hash = source_hash; - } else { - debug_assert_eq!(source_hash, self.source_hash); - } - } - - /// Adds a code region to be counted by an injected counter intrinsic. - pub fn add_counter(&mut self, id: CounterValueReference, region: CodeRegion) { - if let Some(previous_region) = self.counters[id].replace(region.clone()) { - assert_eq!(previous_region, region, "add_counter: code region for id changed"); - } - } - - /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other - /// expressions. Expression IDs start from `u32::MAX` and go down, so the range of expression - /// IDs will not overlap with the range of counter IDs. Counters and expressions can be added in - /// any order, and expressions can still be assigned contiguous (though descending) IDs, without - /// knowing what the last counter ID will be. - /// - /// When storing the expression data in the `expressions` vector in the `FunctionCoverage` - /// struct, its vector index is computed, from the given expression ID, by subtracting from - /// `u32::MAX`. - /// - /// Since the expression operands (`lhs` and `rhs`) can reference either counters or - /// expressions, an operand that references an expression also uses its original ID, descending - /// from `u32::MAX`. Theses operands are translated only during code generation, after all - /// counters and expressions have been added. - pub fn add_counter_expression( - &mut self, - expression_id: InjectedExpressionId, - lhs: ExpressionOperandId, - op: Op, - rhs: ExpressionOperandId, - region: Option<CodeRegion>, - ) { - debug!( - "add_counter_expression({:?}, lhs={:?}, op={:?}, rhs={:?} at {:?}", - expression_id, lhs, op, rhs, region - ); - let expression_index = self.expression_index(u32::from(expression_id)); - debug_assert!( - expression_index.as_usize() < self.expressions.len(), - "expression_index {} is out of range for expressions.len() = {} - for {:?}", - expression_index.as_usize(), - self.expressions.len(), - self, - ); - if let Some(previous_expression) = self.expressions[expression_index].replace(Expression { - lhs, - op, - rhs, - region: region.clone(), - }) { - assert_eq!( - previous_expression, - Expression { lhs, op, rhs, region }, - "add_counter_expression: expression for id changed" - ); - } - } - - /// Add a region that will be marked as "unreachable", with a constant "zero counter". - pub fn add_unreachable_region(&mut self, region: CodeRegion) { - self.unreachable_regions.push(region) - } - - /// Return the source hash, generated from the HIR node structure, and used to indicate whether - /// or not the source code structure changed between different compilations. - pub fn source_hash(&self) -> u64 { - self.source_hash - } - - /// Generate an array of CounterExpressions, and an iterator over all `Counter`s and their - /// associated `Regions` (from which the LLVM-specific `CoverageMapGenerator` will create - /// `CounterMappingRegion`s. - pub fn get_expressions_and_counter_regions( - &self, - ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { - assert!( - self.source_hash != 0 || !self.is_used, - "No counters provided the source_hash for used function: {:?}", - self.instance - ); - - let counter_regions = self.counter_regions(); - let (counter_expressions, expression_regions) = self.expressions_with_regions(); - let unreachable_regions = self.unreachable_regions(); - - let counter_regions = - counter_regions.chain(expression_regions.into_iter().chain(unreachable_regions)); - (counter_expressions, counter_regions) - } - - fn counter_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { - self.counters.iter_enumerated().filter_map(|(index, entry)| { - // Option::map() will return None to filter out missing counters. This may happen - // if, for example, a MIR-instrumented counter is removed during an optimization. - entry.as_ref().map(|region| (Counter::counter_value_reference(index), region)) - }) - } - - fn expressions_with_regions( - &self, - ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { - let mut counter_expressions = Vec::with_capacity(self.expressions.len()); - let mut expression_regions = Vec::with_capacity(self.expressions.len()); - let mut new_indexes = IndexVec::from_elem_n(None, self.expressions.len()); - - // This closure converts any `Expression` operand (`lhs` or `rhs` of the `Op::Add` or - // `Op::Subtract` operation) into its native `llvm::coverage::Counter::CounterKind` type - // and value. Operand ID value `0` maps to `CounterKind::Zero`; values in the known range - // of injected LLVM counters map to `CounterKind::CounterValueReference` (and the value - // matches the injected counter index); and any other value is converted into a - // `CounterKind::Expression` with the expression's `new_index`. - // - // Expressions will be returned from this function in a sequential vector (array) of - // `CounterExpression`, so the expression IDs must be mapped from their original, - // potentially sparse set of indexes, originally in reverse order from `u32::MAX`. - // - // An `Expression` as an operand will have already been encountered as an `Expression` with - // operands, so its new_index will already have been generated (as a 1-up index value). - // (If an `Expression` as an operand does not have a corresponding new_index, it was - // probably optimized out, after the expression was injected into the MIR, so it will - // get a `CounterKind::Zero` instead.) - // - // In other words, an `Expression`s at any given index can include other expressions as - // operands, but expression operands can only come from the subset of expressions having - // `expression_index`s lower than the referencing `Expression`. Therefore, it is - // reasonable to look up the new index of an expression operand while the `new_indexes` - // vector is only complete up to the current `ExpressionIndex`. - let id_to_counter = |new_indexes: &IndexSlice< - InjectedExpressionIndex, - Option<MappedExpressionIndex>, - >, - id: ExpressionOperandId| { - if id == ExpressionOperandId::ZERO { - Some(Counter::zero()) - } else if id.index() < self.counters.len() { - debug_assert!( - id.index() > 0, - "ExpressionOperandId indexes for counters are 1-based, but this id={}", - id.index() - ); - // Note: Some codegen-injected Counters may be only referenced by `Expression`s, - // and may not have their own `CodeRegion`s, - let index = CounterValueReference::from(id.index()); - // Note, the conversion to LLVM `Counter` adjusts the index to be zero-based. - Some(Counter::counter_value_reference(index)) - } else { - let index = self.expression_index(u32::from(id)); - self.expressions - .get(index) - .expect("expression id is out of range") - .as_ref() - // If an expression was optimized out, assume it would have produced a count - // of zero. This ensures that expressions dependent on optimized-out - // expressions are still valid. - .map_or(Some(Counter::zero()), |_| new_indexes[index].map(Counter::expression)) - } - }; - - for (original_index, expression) in - self.expressions.iter_enumerated().filter_map(|(original_index, entry)| { - // Option::map() will return None to filter out missing expressions. This may happen - // if, for example, a MIR-instrumented expression is removed during an optimization. - entry.as_ref().map(|expression| (original_index, expression)) - }) - { - let optional_region = &expression.region; - let Expression { lhs, op, rhs, .. } = *expression; - - if let Some(Some((lhs_counter, mut rhs_counter))) = id_to_counter(&new_indexes, lhs) - .map(|lhs_counter| { - id_to_counter(&new_indexes, rhs).map(|rhs_counter| (lhs_counter, rhs_counter)) - }) - { - if lhs_counter.is_zero() && op.is_subtract() { - // The left side of a subtraction was probably optimized out. As an example, - // a branch condition might be evaluated as a constant expression, and the - // branch could be removed, dropping unused counters in the process. - // - // Since counters are unsigned, we must assume the result of the expression - // can be no more and no less than zero. An expression known to evaluate to zero - // does not need to be added to the coverage map. - // - // Coverage test `loops_branches.rs` includes multiple variations of branches - // based on constant conditional (literal `true` or `false`), and demonstrates - // that the expected counts are still correct. - debug!( - "Expression subtracts from zero (assume unreachable): \ - original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", - original_index, lhs, op, rhs, optional_region, - ); - rhs_counter = Counter::zero(); - } - debug_assert!( - lhs_counter.is_zero() - // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` - || ((lhs_counter.zero_based_id() as usize) - <= usize::max(self.counters.len(), self.expressions.len())), - "lhs id={} > both counters.len()={} and expressions.len()={} - ({:?} {:?} {:?})", - lhs_counter.zero_based_id(), - self.counters.len(), - self.expressions.len(), - lhs_counter, - op, - rhs_counter, - ); - - debug_assert!( - rhs_counter.is_zero() - // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` - || ((rhs_counter.zero_based_id() as usize) - <= usize::max(self.counters.len(), self.expressions.len())), - "rhs id={} > both counters.len()={} and expressions.len()={} - ({:?} {:?} {:?})", - rhs_counter.zero_based_id(), - self.counters.len(), - self.expressions.len(), - lhs_counter, - op, - rhs_counter, - ); - - // Both operands exist. `Expression` operands exist in `self.expressions` and have - // been assigned a `new_index`. - let mapped_expression_index = - MappedExpressionIndex::from(counter_expressions.len()); - let expression = CounterExpression::new( - lhs_counter, - match op { - Op::Add => ExprKind::Add, - Op::Subtract => ExprKind::Subtract, - }, - rhs_counter, - ); - debug!( - "Adding expression {:?} = {:?}, region: {:?}", - mapped_expression_index, expression, optional_region - ); - counter_expressions.push(expression); - new_indexes[original_index] = Some(mapped_expression_index); - if let Some(region) = optional_region { - expression_regions.push((Counter::expression(mapped_expression_index), region)); - } - } else { - bug!( - "expression has one or more missing operands \ - original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", - original_index, - lhs, - op, - rhs, - optional_region, - ); - } - } - (counter_expressions, expression_regions.into_iter()) - } - - fn unreachable_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { - self.unreachable_regions.iter().map(|region| (Counter::zero(), region)) - } - - fn expression_index(&self, id_descending_from_max: u32) -> InjectedExpressionIndex { - debug_assert!(id_descending_from_max >= self.counters.len() as u32); - InjectedExpressionIndex::from(u32::MAX - id_descending_from_max) - } -} diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs deleted file mode 100644 index 569fd3f1a..000000000 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod ffi; -pub mod map; diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs index 6297f9134..e91f7b86e 100644 --- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs +++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs @@ -93,8 +93,7 @@ fn push_debuginfo_type_name<'tcx>( Err(e) => { // Computing the layout can still fail here, e.g. if the target architecture // cannot represent the type. See https://github.com/rust-lang/rust/issues/94961. - // FIXME: migrate once `rustc_middle::mir::interpret::InterpError` is translatable. - tcx.sess.fatal(format!("{}", e)); + tcx.sess.emit_fatal(e.into_diagnostic()); } } } else { diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs index cf4893b82..056b4abd2 100644 --- a/compiler/rustc_codegen_ssa/src/errors.rs +++ b/compiler/rustc_codegen_ssa/src/errors.rs @@ -83,6 +83,12 @@ impl IntoDiagnosticArg for DebugArgPath<'_> { } #[derive(Diagnostic)] +#[diag(codegen_ssa_binary_output_to_tty)] +pub struct BinaryOutputToTty { + pub shorthand: &'static str, +} + +#[derive(Diagnostic)] #[diag(codegen_ssa_ignoring_emit_path)] pub struct IgnoringEmitPath { pub extension: String, @@ -336,7 +342,7 @@ pub struct LinkingFailed<'a> { pub linker_path: &'a PathBuf, pub exit_status: ExitStatus, pub command: &'a Command, - pub escaped_output: &'a str, + pub escaped_output: String, } impl IntoDiagnostic<'_> for LinkingFailed<'_> { @@ -345,11 +351,13 @@ impl IntoDiagnostic<'_> for LinkingFailed<'_> { diag.set_arg("linker_path", format!("{}", self.linker_path.display())); diag.set_arg("exit_status", format!("{}", self.exit_status)); - diag.note(format!("{:?}", self.command)).note(self.escaped_output); + let contains_undefined_ref = self.escaped_output.contains("undefined reference to"); + + diag.note(format!("{:?}", self.command)).note(self.escaped_output.to_string()); // Trying to match an error from OS linkers // which by now we have no way to translate. - if self.escaped_output.contains("undefined reference to") { + if contains_undefined_ref { diag.note(fluent::codegen_ssa_extern_funcs_not_found) .note(fluent::codegen_ssa_specify_libraries_to_link) .note(fluent::codegen_ssa_use_cargo_directive); @@ -1015,3 +1023,9 @@ pub struct TargetFeatureSafeTrait { #[label(codegen_ssa_label_def)] pub def: Span, } + +#[derive(Diagnostic)] +#[diag(codegen_ssa_error_creating_remark_dir)] +pub struct ErrorCreatingRemarkDir { + pub error: std::io::Error, +} diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 31854c7f4..be4c81638 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -4,6 +4,7 @@ #![feature(if_let_guard)] #![feature(int_roundings)] #![feature(let_chains)] +#![feature(negative_impls)] #![feature(never_type)] #![feature(strict_provenance)] #![feature(try_blocks)] @@ -47,7 +48,6 @@ pub mod back; pub mod base; pub mod codegen_attrs; pub mod common; -pub mod coverageinfo; pub mod debuginfo; pub mod errors; pub mod glue; diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 3f0b64b11..9d1b3ce82 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1,5 +1,5 @@ use super::operand::OperandRef; -use super::operand::OperandValue::{Immediate, Pair, Ref}; +use super::operand::OperandValue::{Immediate, Pair, Ref, ZeroSized}; use super::place::PlaceRef; use super::{CachedLlbb, FunctionCx, LocalRef}; @@ -79,8 +79,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { lltarget = fx.landing_pad_for(target); } if is_cleanupret { - // MSVC cross-funclet jump - need a trampoline - debug_assert!(base::wants_msvc_seh(fx.cx.tcx().sess)); + // Cross-funclet jump - need a trampoline + debug_assert!(base::wants_new_eh_instructions(fx.cx.tcx().sess)); debug!("llbb_with_cleanup: creating cleanup trampoline for {:?}", target); let name = &format!("{:?}_cleanup_trampoline_{:?}", self.bb, target); let trampoline_llbb = Bx::append_block(fx.cx, fx.llfn, name); @@ -177,9 +177,16 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { mir::UnwindAction::Continue => None, mir::UnwindAction::Unreachable => None, mir::UnwindAction::Terminate => { - if fx.mir[self.bb].is_cleanup && base::wants_msvc_seh(fx.cx.tcx().sess) { - // SEH will abort automatically if an exception tries to + if fx.mir[self.bb].is_cleanup && base::wants_new_eh_instructions(fx.cx.tcx().sess) { + // MSVC SEH will abort automatically if an exception tries to // propagate out from cleanup. + + // FIXME(@mirkootter): For wasm, we currently do not support terminate during + // cleanup, because this requires a few more changes: The current code + // caches the `terminate_block` for each function; funclet based code - however - + // requires a different terminate_block for each funclet + // Until this is implemented, we just do not unwind inside cleanup blocks + None } else { Some(fx.terminate_block()) @@ -427,6 +434,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { assert_eq!(align, op.layout.align.abi, "return place is unaligned!"); llval } + ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"), }; let ty = bx.cast_backend_type(cast_ty); let addr = bx.pointercast(llslot, bx.type_ptr_to(ty)); @@ -615,7 +623,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { AssertKind::MisalignedPointerDereference { ref required, ref found } => { let required = self.codegen_operand(bx, required).immediate(); let found = self.codegen_operand(bx, found).immediate(); - // It's `fn panic_bounds_check(index: usize, len: usize)`, + // It's `fn panic_misaligned_pointer_dereference(required: usize, found: usize)`, // and `#[track_caller]` adds an implicit third argument. (LangItem::PanicMisalignedPointerDereference, vec![required, found, location]) } @@ -862,13 +870,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // promotes any complex rvalues to constants. if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") { if let mir::Operand::Constant(constant) = arg { - let c = self.eval_mir_constant(constant); - let (llval, ty) = self.simd_shuffle_indices( - &bx, - constant.span, - self.monomorphize(constant.ty()), - c, - ); + let (llval, ty) = self.simd_shuffle_indices(&bx, constant); return OperandRef { val: Immediate(llval), layout: bx.layout_of(ty), @@ -1279,7 +1281,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { destination, target, unwind, - from_hir_call: _, + call_source: _, fn_span, } => self.codegen_call_terminator( helper, @@ -1386,6 +1388,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { (llval, align, true) } } + ZeroSized => match arg.mode { + PassMode::Indirect { .. } => { + // Though `extern "Rust"` doesn't pass ZSTs, some ABIs pass + // a pointer for `repr(C)` structs even when empty, so get + // one from an `alloca` (which can be left uninitialized). + let scratch = PlaceRef::alloca(bx, arg.layout); + (scratch.llval, scratch.align, true) + } + _ => bug!("ZST {op:?} wasn't ignored, but was passed with abi {arg:?}"), + }, }; if by_ref && !arg.is_indirect() { @@ -1493,9 +1505,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if let Some(slot) = self.personality_slot { slot } else { - let layout = cx.layout_of( - cx.tcx().mk_tup(&[cx.tcx().mk_mut_ptr(cx.tcx().types.u8), cx.tcx().types.i32]), - ); + let layout = cx.layout_of(Ty::new_tup( + cx.tcx(), + &[Ty::new_mut_ptr(cx.tcx(), cx.tcx().types.u8), cx.tcx().types.i32], + )); let slot = PlaceRef::alloca(bx, layout); self.personality_slot = Some(slot); slot @@ -1517,7 +1530,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // FIXME(eddyb) rename this to `eh_pad_for_uncached`. fn landing_pad_for_uncached(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { let llbb = self.llbb(bb); - if base::wants_msvc_seh(self.cx.sess()) { + if base::wants_new_eh_instructions(self.cx.sess()) { let cleanup_bb = Bx::append_block(self.cx, self.llfn, &format!("funclet_{:?}", bb)); let mut cleanup_bx = Bx::build(self.cx, cleanup_bb); let funclet = cleanup_bx.cleanup_pad(None, &[]); @@ -1576,6 +1589,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // } catch (...) { // bar(); // } + // + // which creates an IR snippet like + // + // cs_terminate: + // %cs = catchswitch within none [%cp_terminate] unwind to caller + // cp_terminate: + // %cp = catchpad within %cs [null, i32 64, null] + // ... + llbb = Bx::append_block(self.cx, self.llfn, "cs_terminate"); let cp_llbb = Bx::append_block(self.cx, self.llfn, "cp_terminate"); @@ -1718,7 +1740,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { IndirectOperand(tmp, index) => { let op = bx.load_operand(tmp); tmp.storage_dead(bx); - self.locals[index] = LocalRef::Operand(op); + self.overwrite_local(index, LocalRef::Operand(op)); self.debug_introduce_local(bx, index); } DirectOperand(index) => { @@ -1733,7 +1755,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { OperandRef::from_immediate_or_packed_pair(bx, llval, ret_abi.layout) }; - self.locals[index] = LocalRef::Operand(op); + self.overwrite_local(index, LocalRef::Operand(op)); self.debug_introduce_local(bx, index); } } diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs index 14fe84a14..babcf9bee 100644 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -5,7 +5,6 @@ use rustc_middle::mir; use rustc_middle::mir::interpret::{ConstValue, ErrorHandled}; use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::{self, Ty}; -use rustc_span::source_map::Span; use rustc_target::abi::Abi; use super::FunctionCx; @@ -59,22 +58,54 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }) } + /// This is a convenience helper for `simd_shuffle_indices`. It has the precondition + /// that the given `constant` is an `ConstantKind::Unevaluated` and must be convertible to + /// a `ValTree`. If you want a more general version of this, talk to `wg-const-eval` on zulip. + pub fn eval_unevaluated_mir_constant_to_valtree( + &self, + constant: &mir::Constant<'tcx>, + ) -> Result<Option<ty::ValTree<'tcx>>, ErrorHandled> { + let uv = match self.monomorphize(constant.literal) { + mir::ConstantKind::Unevaluated(uv, _) => uv.shrink(), + mir::ConstantKind::Ty(c) => match c.kind() { + // A constant that came from a const generic but was then used as an argument to old-style + // simd_shuffle (passing as argument instead of as a generic param). + rustc_type_ir::ConstKind::Value(valtree) => return Ok(Some(valtree)), + other => span_bug!(constant.span, "{other:#?}"), + }, + // We should never encounter `ConstantKind::Val` unless MIR opts (like const prop) evaluate + // a constant and write that value back into `Operand`s. This could happen, but is unlikely. + // Also: all users of `simd_shuffle` are on unstable and already need to take a lot of care + // around intrinsics. For an issue to happen here, it would require a macro expanding to a + // `simd_shuffle` call without wrapping the constant argument in a `const {}` block, but + // the user pass through arbitrary expressions. + // FIXME(oli-obk): replace the magic const generic argument of `simd_shuffle` with a real + // const generic. + other => span_bug!(constant.span, "{other:#?}"), + }; + let uv = self.monomorphize(uv); + self.cx.tcx().const_eval_resolve_for_typeck( + ty::ParamEnv::reveal_all(), + uv, + Some(constant.span), + ) + } + /// process constant containing SIMD shuffle indices pub fn simd_shuffle_indices( &mut self, bx: &Bx, - span: Span, - ty: Ty<'tcx>, - constant: Result<ConstValue<'tcx>, ErrorHandled>, + constant: &mir::Constant<'tcx>, ) -> (Bx::Value, Ty<'tcx>) { - constant + let ty = self.monomorphize(constant.ty()); + let val = self + .eval_unevaluated_mir_constant_to_valtree(constant) + .ok() + .flatten() .map(|val| { let field_ty = ty.builtin_index().unwrap(); - let c = mir::ConstantKind::from_value(val, ty); - let values: Vec<_> = bx - .tcx() - .destructure_mir_constant(ty::ParamEnv::reveal_all(), c) - .fields + let values: Vec<_> = val + .unwrap_branch() .iter() .map(|field| { if let Some(prim) = field.try_to_scalar() { @@ -88,15 +119,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } }) .collect(); - let llval = bx.const_struct(&values, false); - (llval, c.ty()) + bx.const_struct(&values, false) }) - .unwrap_or_else(|_| { - bx.tcx().sess.emit_err(errors::ShuffleIndicesEvaluation { span }); + .unwrap_or_else(|| { + bx.tcx().sess.emit_err(errors::ShuffleIndicesEvaluation { span: constant.span }); // We've errored, so we don't have to produce working code. - let ty = self.monomorphize(ty); let llty = bx.backend_type(bx.layout_of(ty)); - (bx.const_undef(llty), ty) - }) + bx.const_undef(llty) + }); + (val, ty) } } diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs index f1fe49528..ee7046596 100644 --- a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs @@ -1,13 +1,12 @@ use crate::traits::*; -use rustc_middle::mir::coverage::*; use rustc_middle::mir::Coverage; use rustc_middle::mir::SourceScope; use super::FunctionCx; impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage, scope: SourceScope) { + pub fn codegen_coverage(&self, bx: &mut Bx, coverage: &Coverage, scope: SourceScope) { // Determine the instance that coverage data was originally generated for. let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) { self.monomorphize(inlined) @@ -15,41 +14,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.instance }; - let Coverage { kind, code_region } = coverage; - match kind { - CoverageKind::Counter { function_source_hash, id } => { - if bx.set_function_source_hash(instance, function_source_hash) { - // If `set_function_source_hash()` returned true, the coverage map is enabled, - // so continue adding the counter. - if let Some(code_region) = code_region { - // Note: Some counters do not have code regions, but may still be referenced - // from expressions. In that case, don't add the counter to the coverage map, - // but do inject the counter intrinsic. - bx.add_coverage_counter(instance, id, code_region); - } - - let coverageinfo = bx.tcx().coverageinfo(instance.def); - - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_source_hash); - let num_counters = bx.const_u32(coverageinfo.num_counters); - let index = bx.const_u32(id.zero_based_index()); - debug!( - "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", - fn_name, hash, num_counters, index, - ); - bx.instrprof_increment(fn_name, hash, num_counters, index); - } - } - CoverageKind::Expression { id, lhs, op, rhs } => { - bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region); - } - CoverageKind::Unreachable => { - bx.add_coverage_unreachable( - instance, - code_region.expect("unreachable regions always have code regions"), - ); - } - } + // Handle the coverage info in a backend-specific way. + bx.add_coverage(instance, coverage); } } diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index bba2800fb..1ee89b3d5 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -5,6 +5,7 @@ use rustc_middle::mir; use rustc_middle::ty; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::Ty; use rustc_session::config::DebugInfo; use rustc_span::symbol::{kw, Symbol}; use rustc_span::{BytePos, Span}; @@ -248,7 +249,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } fn spill_operand_to_stack( - operand: &OperandRef<'tcx, Bx::Value>, + operand: OperandRef<'tcx, Bx::Value>, name: Option<String>, bx: &mut Bx, ) -> PlaceRef<'tcx, Bx::Value> { @@ -352,6 +353,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.set_var_name(a, &(name.clone() + ".0")); bx.set_var_name(b, &(name.clone() + ".1")); } + OperandValue::ZeroSized => { + // These never have a value to talk about + } }, LocalRef::PendingOperand => {} } @@ -372,7 +376,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return; } - Self::spill_operand_to_stack(operand, name, bx) + Self::spill_operand_to_stack(*operand, name, bx) } LocalRef::Place(place) => *place, @@ -418,9 +422,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let create_alloca = |bx: &mut Bx, place: PlaceRef<'tcx, Bx::Value>, refcount| { // Create a variable which will be a pointer to the actual value - let ptr_ty = bx - .tcx() - .mk_ptr(ty::TypeAndMut { mutbl: mir::Mutability::Mut, ty: place.layout.ty }); + let ptr_ty = Ty::new_ptr( + bx.tcx(), + ty::TypeAndMut { mutbl: mir::Mutability::Mut, ty: place.layout.ty }, + ); let ptr_layout = bx.layout_of(ptr_ty); let alloca = PlaceRef::alloca(bx, ptr_layout); bx.set_var_name(alloca.llval, &format!("{}.ref{}.dbg.spill", var.name, refcount)); @@ -522,8 +527,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; for _ in 0..var.references { - var_ty = - bx.tcx().mk_ptr(ty::TypeAndMut { mutbl: mir::Mutability::Mut, ty: var_ty }); + var_ty = Ty::new_ptr( + bx.tcx(), + ty::TypeAndMut { mutbl: mir::Mutability::Mut, ty: var_ty }, + ); } self.cx.create_dbg_var(var.name, var_ty, dbg_scope, var_kind, span) @@ -547,7 +554,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if let Ok(operand) = self.eval_mir_constant_to_operand(bx, &c) { self.set_debug_loc(bx, var.source_info); let base = Self::spill_operand_to_stack( - &operand, + operand, Some(var.name.to_string()), bx, ); diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 1479242f2..8a65dd593 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -211,68 +211,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { args[1].val.unaligned_volatile_store(bx, dst); return; } - | sym::unchecked_div - | sym::unchecked_rem - | sym::unchecked_shl - | sym::unchecked_shr - | sym::unchecked_add - | sym::unchecked_sub - | sym::unchecked_mul - | sym::exact_div => { + sym::exact_div => { let ty = arg_tys[0]; match int_type_width_signed(ty, bx.tcx()) { - Some((_width, signed)) => match name { - sym::exact_div => { - if signed { - bx.exactsdiv(args[0].immediate(), args[1].immediate()) - } else { - bx.exactudiv(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_div => { - if signed { - bx.sdiv(args[0].immediate(), args[1].immediate()) - } else { - bx.udiv(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_rem => { - if signed { - bx.srem(args[0].immediate(), args[1].immediate()) - } else { - bx.urem(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_shl => bx.shl(args[0].immediate(), args[1].immediate()), - sym::unchecked_shr => { - if signed { - bx.ashr(args[0].immediate(), args[1].immediate()) - } else { - bx.lshr(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_add => { - if signed { - bx.unchecked_sadd(args[0].immediate(), args[1].immediate()) - } else { - bx.unchecked_uadd(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_sub => { - if signed { - bx.unchecked_ssub(args[0].immediate(), args[1].immediate()) - } else { - bx.unchecked_usub(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_mul => { - if signed { - bx.unchecked_smul(args[0].immediate(), args[1].immediate()) - } else { - bx.unchecked_umul(args[0].immediate(), args[1].immediate()) - } + Some((_width, signed)) => { + if signed { + bx.exactsdiv(args[0].immediate(), args[1].immediate()) + } else { + bx.exactudiv(args[0].immediate(), args[1].immediate()) } - _ => bug!(), }, None => { bx.tcx().sess.emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty }); diff --git a/compiler/rustc_codegen_ssa/src/mir/locals.rs b/compiler/rustc_codegen_ssa/src/mir/locals.rs new file mode 100644 index 000000000..378c54013 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/locals.rs @@ -0,0 +1,75 @@ +//! Locals are in a private module as updating `LocalRef::Operand` has to +//! be careful wrt to subtyping. To deal with this we only allow updates by using +//! `FunctionCx::overwrite_local` which handles it automatically. +use crate::mir::{FunctionCx, LocalRef}; +use crate::traits::BuilderMethods; +use rustc_index::IndexVec; +use rustc_middle::mir; +use rustc_middle::ty::print::with_no_trimmed_paths; +use std::ops::{Index, IndexMut}; +pub(super) struct Locals<'tcx, V> { + values: IndexVec<mir::Local, LocalRef<'tcx, V>>, +} + +impl<'tcx, V> Index<mir::Local> for Locals<'tcx, V> { + type Output = LocalRef<'tcx, V>; + #[inline] + fn index(&self, index: mir::Local) -> &LocalRef<'tcx, V> { + &self.values[index] + } +} + +/// To mutate locals, use `FunctionCx::overwrite_local` instead. +impl<'tcx, V, Idx: ?Sized> !IndexMut<Idx> for Locals<'tcx, V> {} + +impl<'tcx, V> Locals<'tcx, V> { + pub(super) fn empty() -> Locals<'tcx, V> { + Locals { values: IndexVec::default() } + } + + pub(super) fn indices(&self) -> impl DoubleEndedIterator<Item = mir::Local> + Clone + 'tcx { + self.values.indices() + } +} + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + pub(super) fn initialize_locals(&mut self, values: Vec<LocalRef<'tcx, Bx::Value>>) { + assert!(self.locals.values.is_empty()); + // FIXME(#115215): After #115025 get's merged this might not be necessary + for (local, value) in values.into_iter().enumerate() { + match value { + LocalRef::Place(_) | LocalRef::UnsizedPlace(_) | LocalRef::PendingOperand => (), + LocalRef::Operand(op) => { + let local = mir::Local::from_usize(local); + let expected_ty = self.monomorphize(self.mir.local_decls[local].ty); + if expected_ty != op.layout.ty { + warn!("Unexpected initial operand type. See the issues/114858"); + } + } + } + self.locals.values.push(value); + } + } + + pub(super) fn overwrite_local( + &mut self, + local: mir::Local, + mut value: LocalRef<'tcx, Bx::Value>, + ) { + match value { + LocalRef::Place(_) | LocalRef::UnsizedPlace(_) | LocalRef::PendingOperand => (), + LocalRef::Operand(ref mut op) => { + let local_ty = self.monomorphize(self.mir.local_decls[local].ty); + if local_ty != op.layout.ty { + // FIXME(#112651): This can be changed to an ICE afterwards. + debug!("updating type of operand due to subtyping"); + with_no_trimmed_paths!(debug!(?op.layout.ty)); + with_no_trimmed_paths!(debug!(?local_ty)); + op.layout.ty = local_ty; + } + } + }; + + self.locals.values[local] = value; + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 1204c99e5..9ff6a2497 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -1,21 +1,31 @@ use crate::base; use crate::traits::*; +use rustc_index::bit_set::BitSet; +use rustc_index::IndexVec; use rustc_middle::mir; use rustc_middle::mir::interpret::ErrorHandled; +use rustc_middle::mir::traversal; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt}; use rustc_target::abi::call::{FnAbi, PassMode}; use std::iter; -use rustc_index::bit_set::BitSet; -use rustc_index::IndexVec; +mod analyze; +mod block; +pub mod constant; +pub mod coverageinfo; +pub mod debuginfo; +mod intrinsic; +mod locals; +pub mod operand; +pub mod place; +mod rvalue; +mod statement; use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo}; -use self::place::PlaceRef; -use rustc_middle::mir::traversal; - use self::operand::{OperandRef, OperandValue}; +use self::place::PlaceRef; // Used for tracking the state of generated basic blocks. enum CachedLlbb<T> { @@ -91,7 +101,7 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { /// /// Avoiding allocs can also be important for certain intrinsics, /// notably `expect`. - locals: IndexVec<mir::Local, LocalRef<'tcx, Bx::Value>>, + locals: locals::Locals<'tcx, Bx::Value>, /// All `VarDebugInfo` from the MIR body, partitioned by `Local`. /// This is `None` if no var`#[non_exhaustive]`iable debuginfo/names are needed. @@ -111,7 +121,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.instance.subst_mir_and_normalize_erasing_regions( self.cx.tcx(), ty::ParamEnv::reveal_all(), - ty::EarlyBinder(value), + ty::EarlyBinder::bind(value), ) } } @@ -129,16 +139,13 @@ enum LocalRef<'tcx, V> { PendingOperand, } -impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> { - fn new_operand<Bx: BuilderMethods<'a, 'tcx, Value = V>>( - bx: &mut Bx, - layout: TyAndLayout<'tcx>, - ) -> LocalRef<'tcx, V> { +impl<'tcx, V: CodegenObject> LocalRef<'tcx, V> { + fn new_operand(layout: TyAndLayout<'tcx>) -> LocalRef<'tcx, V> { if layout.is_zst() { // Zero-size temporaries aren't always initialized, which // doesn't matter because they don't contain data, but // we need something in the operand. - LocalRef::Operand(OperandRef::new_zst(bx, layout)) + LocalRef::Operand(OperandRef::zero_sized(layout)) } else { LocalRef::PendingOperand } @@ -172,7 +179,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( start_bx.set_personality_fn(cx.eh_personality()); } - let cleanup_kinds = base::wants_msvc_seh(cx.tcx().sess).then(|| analyze::cleanup_kinds(&mir)); + let cleanup_kinds = + base::wants_new_eh_instructions(cx.tcx().sess).then(|| analyze::cleanup_kinds(&mir)); let cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>> = mir.basic_blocks @@ -195,7 +203,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( cleanup_kinds, landing_pads: IndexVec::from_elem(None, &mir.basic_blocks), funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()), - locals: IndexVec::new(), + locals: locals::Locals::empty(), debug_context, per_local_var_debug_info: None, caller_location: None, @@ -226,7 +234,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let memory_locals = analyze::non_ssa_locals(&fx); // Allocate variable and temp allocas - fx.locals = { + let local_values = { let args = arg_local_refs(&mut start_bx, &mut fx, &memory_locals); let mut allocate_local = |local| { @@ -249,7 +257,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } else { debug!("alloc: {:?} -> operand", local); - LocalRef::new_operand(&mut start_bx, layout) + LocalRef::new_operand(layout) } }; @@ -259,6 +267,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( .chain(mir.vars_and_temps_iter().map(allocate_local)) .collect() }; + fx.initialize_locals(local_values); // Apply debuginfo to the newly allocated locals. fx.debug_introduce_locals(&mut start_bx); @@ -292,14 +301,13 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( .enumerate() .map(|(arg_index, local)| { let arg_decl = &mir.local_decls[local]; + let arg_ty = fx.monomorphize(arg_decl.ty); if Some(local) == mir.spread_arg { // This argument (e.g., the last argument in the "rust-call" ABI) // is a tuple that was spread at the ABI level and now we have // to reconstruct it into a tuple local variable, from multiple // individual LLVM function arguments. - - let arg_ty = fx.monomorphize(arg_decl.ty); let ty::Tuple(tupled_arg_tys) = arg_ty.kind() else { bug!("spread argument isn't a tuple?!"); }; @@ -334,8 +342,6 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } if fx.fn_abi.c_variadic && arg_index == fx.fn_abi.args.len() { - let arg_ty = fx.monomorphize(arg_decl.ty); - let va_list = PlaceRef::alloca(bx, bx.layout_of(arg_ty)); bx.va_start(va_list.llval); @@ -355,7 +361,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let local = |op| LocalRef::Operand(op); match arg.mode { PassMode::Ignore => { - return local(OperandRef::new_zst(bx, arg.layout)); + return local(OperandRef::zero_sized(arg.layout)); } PassMode::Direct(_) => { let llarg = bx.get_param(llarg_idx); @@ -432,14 +438,3 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( args } - -mod analyze; -mod block; -pub mod constant; -pub mod coverageinfo; -pub mod debuginfo; -mod intrinsic; -pub mod operand; -pub mod place; -mod rvalue; -mod statement; diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 2301c3ef1..31c293d7c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -8,10 +8,10 @@ use crate::traits::*; use crate::MemFlags; use rustc_middle::mir; -use rustc_middle::mir::interpret::{ConstValue, Pointer, Scalar}; +use rustc_middle::mir::interpret::{alloc_range, ConstValue, Pointer, Scalar}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::Ty; -use rustc_target::abi::{Abi, Align, Size}; +use rustc_target::abi::{self, Abi, Align, Size}; use std::fmt; @@ -45,6 +45,14 @@ pub enum OperandValue<V> { /// as returned by [`LayoutTypeMethods::scalar_pair_element_backend_type`] /// with `immediate: true`. Pair(V, V), + /// A value taking no bytes, and which therefore needs no LLVM value at all. + /// + /// If you ever need a `V` to pass to something, get a fresh poison value + /// from [`ConstMethods::const_poison`]. + /// + /// An `OperandValue` *must* be this variant for any type for which + /// `is_zst` on its `Layout` returns `true`. + ZeroSized, } /// An `OperandRef` is an "SSA" reference to a Rust value, along with @@ -71,15 +79,9 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> { } impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { - pub fn new_zst<Bx: BuilderMethods<'a, 'tcx, Value = V>>( - bx: &mut Bx, - layout: TyAndLayout<'tcx>, - ) -> OperandRef<'tcx, V> { + pub fn zero_sized(layout: TyAndLayout<'tcx>) -> OperandRef<'tcx, V> { assert!(layout.is_zst()); - OperandRef { - val: OperandValue::Immediate(bx.const_poison(bx.immediate_backend_type(layout))), - layout, - } + OperandRef { val: OperandValue::ZeroSized, layout } } pub fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>( @@ -97,7 +99,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout)); OperandValue::Immediate(llval) } - ConstValue::ZeroSized => return OperandRef::new_zst(bx, layout), + ConstValue::ZeroSized => return OperandRef::zero_sized(layout), ConstValue::Slice { data, start, end } => { let Abi::ScalarPair(a_scalar, _) = layout.abi else { bug!("from_const: invalid ScalarPair layout: {:#?}", layout); @@ -115,13 +117,82 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { OperandValue::Pair(a_llval, b_llval) } ConstValue::ByRef { alloc, offset } => { - return bx.load_operand(bx.from_const_alloc(layout, alloc, offset)); + return Self::from_const_alloc(bx, layout, alloc, offset); } }; OperandRef { val, layout } } + fn from_const_alloc<Bx: BuilderMethods<'a, 'tcx, Value = V>>( + bx: &mut Bx, + layout: TyAndLayout<'tcx>, + alloc: rustc_middle::mir::interpret::ConstAllocation<'tcx>, + offset: Size, + ) -> Self { + let alloc_align = alloc.inner().align; + assert_eq!(alloc_align, layout.align.abi); + let ty = bx.type_ptr_to(bx.cx().backend_type(layout)); + + let read_scalar = |start, size, s: abi::Scalar, ty| { + let val = alloc + .0 + .read_scalar( + bx, + alloc_range(start, size), + /*read_provenance*/ matches!(s.primitive(), abi::Pointer(_)), + ) + .unwrap(); + bx.scalar_to_backend(val, s, ty) + }; + + // It may seem like all types with `Scalar` or `ScalarPair` ABI are fair game at this point. + // However, `MaybeUninit<u64>` is considered a `Scalar` as far as its layout is concerned -- + // and yet cannot be represented by an interpreter `Scalar`, since we have to handle the + // case where some of the bytes are initialized and others are not. So, we need an extra + // check that walks over the type of `mplace` to make sure it is truly correct to treat this + // like a `Scalar` (or `ScalarPair`). + match layout.abi { + Abi::Scalar(s @ abi::Scalar::Initialized { .. }) => { + let size = s.size(bx); + assert_eq!(size, layout.size, "abi::Scalar size does not match layout size"); + let val = read_scalar(Size::ZERO, size, s, ty); + OperandRef { val: OperandValue::Immediate(val), layout } + } + Abi::ScalarPair( + a @ abi::Scalar::Initialized { .. }, + b @ abi::Scalar::Initialized { .. }, + ) => { + let (a_size, b_size) = (a.size(bx), b.size(bx)); + let b_offset = a_size.align_to(b.align(bx).abi); + assert!(b_offset.bytes() > 0); + let a_val = read_scalar( + Size::ZERO, + a_size, + a, + bx.scalar_pair_element_backend_type(layout, 0, true), + ); + let b_val = read_scalar( + b_offset, + b_size, + b, + bx.scalar_pair_element_backend_type(layout, 1, true), + ); + OperandRef { val: OperandValue::Pair(a_val, b_val), layout } + } + _ if layout.is_zst() => OperandRef::zero_sized(layout), + _ => { + // Neither a scalar nor scalar pair. Load from a place + let init = bx.const_data_from_alloc(alloc); + let base_addr = bx.static_addr_of(init, alloc_align, None); + + let llval = bx.const_ptr_byte_offset(base_addr, offset); + let llval = bx.const_bitcast(llval, ty); + bx.load_operand(PlaceRef::new_sized(llval, layout)) + } + } + } + /// Asserts that this operand refers to a scalar and returns /// a reference to its value. pub fn immediate(self) -> V { @@ -147,6 +218,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { OperandValue::Immediate(llptr) => (llptr, None), OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)), OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self), + OperandValue::ZeroSized => bug!("Deref of ZST operand {:?}", self), }; let layout = cx.layout_of(projected_ty); PlaceRef { llval: llptr, llextra, layout, align: layout.align.abi } @@ -204,9 +276,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let mut val = match (self.val, self.layout.abi) { // If the field is ZST, it has no data. - _ if field.is_zst() => { - return OperandRef::new_zst(bx, field); - } + _ if field.is_zst() => OperandValue::ZeroSized, // Newtype of a scalar, scalar pair or vector. (OperandValue::Immediate(_) | OperandValue::Pair(..), _) @@ -237,6 +307,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { }; match (&mut val, field.abi) { + (OperandValue::ZeroSized, _) => {} ( OperandValue::Immediate(llval), Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. }, @@ -290,8 +361,8 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { impl<'a, 'tcx, V: CodegenObject> OperandValue<V> { /// Returns an `OperandValue` that's generally UB to use in any way. /// - /// Depending on the `layout`, returns an `Immediate` or `Pair` containing - /// poison value(s), or a `Ref` containing a poison pointer. + /// Depending on the `layout`, returns `ZeroSized` for ZSTs, an `Immediate` or + /// `Pair` containing poison value(s), or a `Ref` containing a poison pointer. /// /// Supports sized types only. pub fn poison<Bx: BuilderMethods<'a, 'tcx, Value = V>>( @@ -299,7 +370,9 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> { layout: TyAndLayout<'tcx>, ) -> OperandValue<V> { assert!(layout.is_sized()); - if bx.cx().is_backend_immediate(layout) { + if layout.is_zst() { + OperandValue::ZeroSized + } else if bx.cx().is_backend_immediate(layout) { let ibty = bx.cx().immediate_backend_type(layout); OperandValue::Immediate(bx.const_poison(ibty)) } else if bx.cx().is_backend_scalar_pair(layout) { @@ -352,12 +425,11 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> { flags: MemFlags, ) { debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest); - // Avoid generating stores of zero-sized values, because the only way to have a zero-sized - // value is through `undef`, and store itself is useless. - if dest.layout.is_zst() { - return; - } match self { + OperandValue::ZeroSized => { + // Avoid generating stores of zero-sized values, because the only way to have a zero-sized + // value is through `undef`/`poison`, and the store itself is useless. + } OperandValue::Ref(r, None, source_align) => { if flags.contains(MemFlags::NONTEMPORAL) { // HACK(nox): This is inefficient but there is no nontemporal memcpy. @@ -458,7 +530,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // checks in `codegen_consume` and `extract_field`. let elem = o.layout.field(bx.cx(), 0); if elem.is_zst() { - o = OperandRef::new_zst(bx, elem); + o = OperandRef::zero_sized(elem); } else { return None; } @@ -492,7 +564,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // ZSTs don't require any actual memory access. if layout.is_zst() { - return OperandRef::new_zst(bx, layout); + return OperandRef::zero_sized(layout); } if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) { diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index a58a61cd5..ab493ae5c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -61,7 +61,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { layout: TyAndLayout<'tcx>, ) -> Self { assert!(layout.is_unsized(), "tried to allocate indirect place for sized values"); - let ptr_ty = bx.cx().tcx().mk_mut_ptr(layout.ty); + let ptr_ty = Ty::new_mut_ptr(bx.cx().tcx(), layout.ty); let ptr_layout = bx.cx().layout_of(ptr_ty); Self::alloca(bx, ptr_layout) } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 6e7065713..956f03d25 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -11,7 +11,7 @@ use rustc_middle::mir; use rustc_middle::mir::Operand; use rustc_middle::ty::cast::{CastTy, IntTy}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; -use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt}; +use rustc_middle::ty::{self, adjustment::PointerCoercion, Instance, Ty, TyCtxt}; use rustc_session::config::OptLevel; use rustc_span::source_map::{Span, DUMMY_SP}; use rustc_target::abi::{self, FIRST_VARIANT}; @@ -32,7 +32,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { cg_operand.val.store(bx, dest); } - mir::Rvalue::Cast(mir::CastKind::Pointer(PointerCast::Unsize), ref source, _) => { + mir::Rvalue::Cast( + mir::CastKind::PointerCoercion(PointerCoercion::Unsize), + ref source, + _, + ) => { // The destination necessarily contains a fat pointer, so if // it's a scalar pair, it's a fat pointer or newtype thereof. if bx.cx().is_backend_scalar_pair(dest.layout) { @@ -70,6 +74,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::Ref(_, Some(_), _) => { bug!("unsized coercion on an unsized rvalue"); } + OperandValue::ZeroSized => { + bug!("unsized coercion on a ZST rvalue"); + } } } @@ -165,11 +172,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } match src.val { - OperandValue::Ref(..) => { + OperandValue::Ref(..) | OperandValue::ZeroSized => { span_bug!( self.mir.span, "Operand path should have handled transmute \ - from `Ref` {src:?} to place {dst:?}" + from {src:?} to place {dst:?}" ); } OperandValue::Immediate(..) | OperandValue::Pair(..) => { @@ -220,17 +227,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let fake_place = PlaceRef::new_sized_aligned(cast_ptr, cast, align); Some(bx.load_operand(fake_place).val) } + OperandValue::ZeroSized => { + let OperandValueKind::ZeroSized = operand_kind else { + bug!("Found {operand_kind:?} for operand {operand:?}"); + }; + if let OperandValueKind::ZeroSized = cast_kind { + Some(OperandValue::ZeroSized) + } else { + None + } + } OperandValue::Immediate(imm) => { let OperandValueKind::Immediate(in_scalar) = operand_kind else { bug!("Found {operand_kind:?} for operand {operand:?}"); }; - if let OperandValueKind::Immediate(out_scalar) = cast_kind { - match (in_scalar, out_scalar) { - (ScalarOrZst::Zst, ScalarOrZst::Zst) => { - Some(OperandRef::new_zst(bx, cast).val) - } - (ScalarOrZst::Scalar(in_scalar), ScalarOrZst::Scalar(out_scalar)) - if in_scalar.size(self.cx) == out_scalar.size(self.cx) => + if let OperandValueKind::Immediate(out_scalar) = cast_kind + && in_scalar.size(self.cx) == out_scalar.size(self.cx) { let operand_bty = bx.backend_type(operand.layout); let cast_bty = bx.backend_type(cast); @@ -242,9 +254,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { out_scalar, cast_bty, ))) - } - _ => None, - } } else { None } @@ -406,7 +415,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let lladdr = bx.ptrtoint(llptr, llcast_ty); OperandValue::Immediate(lladdr) } - mir::CastKind::Pointer(PointerCast::ReifyFnPointer) => { + mir::CastKind::PointerCoercion(PointerCoercion::ReifyFnPointer) => { match *operand.layout.ty.kind() { ty::FnDef(def_id, substs) => { let instance = ty::Instance::resolve_for_fn_ptr( @@ -422,7 +431,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => bug!("{} cannot be reified to a fn ptr", operand.layout.ty), } } - mir::CastKind::Pointer(PointerCast::ClosureFnPointer(_)) => { + mir::CastKind::PointerCoercion(PointerCoercion::ClosureFnPointer(_)) => { match *operand.layout.ty.kind() { ty::Closure(def_id, substs) => { let instance = Instance::resolve_closure( @@ -438,11 +447,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => bug!("{} cannot be cast to a fn ptr", operand.layout.ty), } } - mir::CastKind::Pointer(PointerCast::UnsafeFnPointer) => { + mir::CastKind::PointerCoercion(PointerCoercion::UnsafeFnPointer) => { // This is a no-op at the LLVM level. operand.val } - mir::CastKind::Pointer(PointerCast::Unsize) => { + mir::CastKind::PointerCoercion(PointerCoercion::Unsize) => { assert!(bx.cx().is_backend_scalar_pair(cast)); let (lldata, llextra) = match operand.val { OperandValue::Pair(lldata, llextra) => { @@ -457,12 +466,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::Ref(..) => { bug!("by-ref operand {:?} in `codegen_rvalue_operand`", operand); } + OperandValue::ZeroSized => { + bug!("zero-sized operand {:?} in `codegen_rvalue_operand`", operand); + } }; let (lldata, llextra) = base::unsize_ptr(bx, lldata, operand.layout.ty, cast.ty, llextra); OperandValue::Pair(lldata, llextra) } - mir::CastKind::Pointer(PointerCast::MutToConstPointer) + mir::CastKind::PointerCoercion(PointerCoercion::MutToConstPointer) | mir::CastKind::PtrToPtr if bx.cx().is_backend_scalar_pair(operand.layout) => { @@ -490,13 +502,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::Ref(_, _, _) => todo!(), OperandValue::Immediate(v) => (v, None), OperandValue::Pair(v, l) => (v, Some(l)), + OperandValue::ZeroSized => bug!("ZST -- which is not PointerLike -- in DynStar"), }; let (lldata, llextra) = base::cast_to_dyn_star(bx, lldata, operand.layout, cast.ty, llextra); OperandValue::Pair(lldata, llextra) } - mir::CastKind::Pointer( - PointerCast::MutToConstPointer | PointerCast::ArrayToPointer, + mir::CastKind::PointerCoercion( + PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer, ) | mir::CastKind::IntToInt | mir::CastKind::FloatToInt @@ -572,7 +585,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::Ref(_, bk, place) => { let mk_ref = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { - tcx.mk_ref( + Ty::new_ref( + tcx, tcx.lifetimes.re_erased, ty::TypeAndMut { ty, mutbl: bk.to_mutbl_lossy() }, ) @@ -583,7 +597,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::CopyForDeref(place) => self.codegen_operand(bx, &Operand::Copy(place)), mir::Rvalue::AddressOf(mutability, place) => { let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { - tcx.mk_ptr(ty::TypeAndMut { ty, mutbl: mutability }) + Ty::new_ptr(tcx, ty::TypeAndMut { ty, mutbl: mutability }) }; self.codegen_place_to_pointer(bx, place, mk_ptr) } @@ -635,7 +649,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { lhs.layout.ty, ); let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty); - let operand_ty = bx.tcx().mk_tup(&[val_ty, bx.tcx().types.bool]); + let operand_ty = Ty::new_tup(bx.tcx(), &[val_ty, bx.tcx().types.bool]); OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) } } @@ -668,11 +682,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::NullaryOp(ref null_op, ty) => { let ty = self.monomorphize(ty); - assert!(bx.cx().type_is_sized(ty)); let layout = bx.cx().layout_of(ty); let val = match null_op { - mir::NullOp::SizeOf => layout.size.bytes(), - mir::NullOp::AlignOf => layout.align.abi.bytes(), + mir::NullOp::SizeOf => { + assert!(bx.cx().type_is_sized(ty)); + layout.size.bytes() + } + mir::NullOp::AlignOf => { + assert!(bx.cx().type_is_sized(ty)); + layout.align.abi.bytes() + } mir::NullOp::OffsetOf(fields) => { layout.offset_of_subfield(bx.cx(), fields.iter().map(|f| f.index())).bytes() } @@ -713,14 +732,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // According to `rvalue_creates_operand`, only ZST // aggregate rvalues are allowed to be operands. let ty = rvalue.ty(self.mir, self.cx.tcx()); - OperandRef::new_zst(bx, self.cx.layout_of(self.monomorphize(ty))) + OperandRef::zero_sized(self.cx.layout_of(self.monomorphize(ty))) } mir::Rvalue::ShallowInitBox(ref operand, content_ty) => { let operand = self.codegen_operand(bx, operand); let lloperand = operand.immediate(); let content_ty = self.monomorphize(content_ty); - let box_layout = bx.cx().layout_of(bx.tcx().mk_box(content_ty)); + let box_layout = bx.cx().layout_of(Ty::new_box(bx.tcx(), content_ty)); let llty_ptr = bx.cx().backend_type(box_layout); let val = bx.pointercast(lloperand, llty_ptr); @@ -784,6 +803,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.add(lhs, rhs) } } + mir::BinOp::AddUnchecked => { + if is_signed { + bx.unchecked_sadd(lhs, rhs) + } else { + bx.unchecked_uadd(lhs, rhs) + } + } mir::BinOp::Sub => { if is_float { bx.fsub(lhs, rhs) @@ -791,6 +817,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.sub(lhs, rhs) } } + mir::BinOp::SubUnchecked => { + if is_signed { + bx.unchecked_ssub(lhs, rhs) + } else { + bx.unchecked_usub(lhs, rhs) + } + } mir::BinOp::Mul => { if is_float { bx.fmul(lhs, rhs) @@ -798,6 +831,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.mul(lhs, rhs) } } + mir::BinOp::MulUnchecked => { + if is_signed { + bx.unchecked_smul(lhs, rhs) + } else { + bx.unchecked_umul(lhs, rhs) + } + } mir::BinOp::Div => { if is_float { bx.fdiv(lhs, rhs) @@ -834,8 +874,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.inbounds_gep(llty, lhs, &[rhs]) } } - mir::BinOp::Shl => common::build_unchecked_lshift(bx, lhs, rhs), - mir::BinOp::Shr => common::build_unchecked_rshift(bx, input_ty, lhs, rhs), + mir::BinOp::Shl => common::build_masked_lshift(bx, lhs, rhs), + mir::BinOp::ShlUnchecked => { + let rhs = base::cast_shift_expr_rhs(bx, lhs, rhs); + bx.shl(lhs, rhs) + } + mir::BinOp::Shr => common::build_masked_rshift(bx, input_ty, lhs, rhs), + mir::BinOp::ShrUnchecked => { + let rhs = base::cast_shift_expr_rhs(bx, lhs, rhs); + if is_signed { bx.ashr(lhs, rhs) } else { bx.lshr(lhs, rhs) } + } mir::BinOp::Ne | mir::BinOp::Lt | mir::BinOp::Gt @@ -931,6 +979,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Can always load from a pointer as needed (OperandValueKind::Ref, _) => true, + // ZST-to-ZST is the easiest thing ever + (OperandValueKind::ZeroSized, OperandValueKind::ZeroSized) => true, + + // But if only one of them is a ZST the sizes can't match + (OperandValueKind::ZeroSized, _) | (_, OperandValueKind::ZeroSized) => false, + // Need to generate an `alloc` to get a pointer from an immediate (OperandValueKind::Immediate(..) | OperandValueKind::Pair(..), OperandValueKind::Ref) => false, @@ -974,12 +1028,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { /// Gets which variant of [`OperandValue`] is expected for a particular type. fn value_kind(&self, layout: TyAndLayout<'tcx>) -> OperandValueKind { - if self.cx.is_backend_immediate(layout) { + if layout.is_zst() { + OperandValueKind::ZeroSized + } else if self.cx.is_backend_immediate(layout) { debug_assert!(!self.cx.is_backend_scalar_pair(layout)); OperandValueKind::Immediate(match layout.abi { - abi::Abi::Scalar(s) => ScalarOrZst::Scalar(s), - abi::Abi::Vector { element, .. } => ScalarOrZst::Scalar(element), - _ if layout.is_zst() => ScalarOrZst::Zst, + abi::Abi::Scalar(s) => s, + abi::Abi::Vector { element, .. } => element, x => span_bug!(self.mir.span, "Couldn't translate {x:?} as backend immediate"), }) } else if self.cx.is_backend_scalar_pair(layout) { @@ -1002,21 +1057,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { #[derive(Debug, Copy, Clone)] enum OperandValueKind { Ref, - Immediate(ScalarOrZst), + Immediate(abi::Scalar), Pair(abi::Scalar, abi::Scalar), -} - -#[derive(Debug, Copy, Clone)] -enum ScalarOrZst { - Zst, - Scalar(abi::Scalar), -} - -impl ScalarOrZst { - pub fn size(self, cx: &impl abi::HasDataLayout) -> abi::Size { - match self { - ScalarOrZst::Zst => abi::Size::ZERO, - ScalarOrZst::Scalar(s) => s.size(cx), - } - } + ZeroSized, } diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index 3fd7397ad..899e41265 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -20,7 +20,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } LocalRef::PendingOperand => { let operand = self.codegen_rvalue_operand(bx, rvalue); - self.locals[index] = LocalRef::Operand(operand); + self.overwrite_local(index, LocalRef::Operand(operand)); self.debug_introduce_local(bx, index); } LocalRef::Operand(op) => { @@ -65,7 +65,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } mir::StatementKind::Coverage(box ref coverage) => { - self.codegen_coverage(bx, coverage.clone(), statement.source_info.scope); + self.codegen_coverage(bx, coverage, statement.source_info.scope); } mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => { let op_val = self.codegen_operand(bx, op); diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index c5976a654..9e06fec55 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -44,6 +44,7 @@ const ARM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[ // #[target_feature]. ("thumb-mode", Some(sym::arm_target_feature)), ("thumb2", Some(sym::arm_target_feature)), + ("trustzone", Some(sym::arm_target_feature)), ("v5te", Some(sym::arm_target_feature)), ("v6", Some(sym::arm_target_feature)), ("v6k", Some(sym::arm_target_feature)), @@ -53,6 +54,7 @@ const ARM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[ ("vfp2", Some(sym::arm_target_feature)), ("vfp3", Some(sym::arm_target_feature)), ("vfp4", Some(sym::arm_target_feature)), + ("virtualization", Some(sym::arm_target_feature)), // tidy-alphabetical-end ]; @@ -282,6 +284,7 @@ const WASM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[ // tidy-alphabetical-start ("atomics", Some(sym::wasm_target_feature)), ("bulk-memory", Some(sym::wasm_target_feature)), + ("exception-handling", Some(sym::wasm_target_feature)), ("multivalue", Some(sym::wasm_target_feature)), ("mutable-globals", Some(sym::wasm_target_feature)), ("nontrapping-fptoint", Some(sym::wasm_target_feature)), diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs index d83bfc740..b3c9ecf8b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/backend.rs +++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs @@ -6,7 +6,7 @@ use crate::back::write::TargetMachineFactoryFn; use crate::{CodegenResults, ModuleCodegen}; use rustc_ast::expand::allocator::AllocatorKind; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::sync::{DynSend, DynSync}; use rustc_errors::ErrorGuaranteed; use rustc_metadata::EncodedMetadata; @@ -101,7 +101,7 @@ pub trait CodegenBackend { ongoing_codegen: Box<dyn Any>, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed>; + ) -> Result<(CodegenResults, FxIndexMap<WorkProductId, WorkProduct>), ErrorGuaranteed>; /// This is called on the returned `Box<dyn Any>` from `join_codegen` /// diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs index 619063027..d6e9bfce1 100644 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -1,8 +1,6 @@ use super::BackendTypes; -use crate::mir::place::PlaceRef; use rustc_middle::mir::interpret::{ConstAllocation, Scalar}; -use rustc_middle::ty::layout::TyAndLayout; -use rustc_target::abi::{self, Size}; +use rustc_target::abi; pub trait ConstMethods<'tcx>: BackendTypes { // Constant constructors @@ -17,6 +15,7 @@ pub trait ConstMethods<'tcx>: BackendTypes { fn const_i32(&self, i: i32) -> Self::Value; fn const_u32(&self, i: u32) -> Self::Value; fn const_u64(&self, i: u64) -> Self::Value; + fn const_u128(&self, i: u128) -> Self::Value; fn const_usize(&self, i: u64) -> Self::Value; fn const_u8(&self, i: u8) -> Self::Value; fn const_real(&self, t: Self::Type, val: f64) -> Self::Value; @@ -30,12 +29,8 @@ pub trait ConstMethods<'tcx>: BackendTypes { fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value; fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value; - fn from_const_alloc( - &self, - layout: TyAndLayout<'tcx>, - alloc: ConstAllocation<'tcx>, - offset: Size, - ) -> PlaceRef<'tcx, Self::Value>; fn const_ptrcast(&self, val: Self::Value, ty: Self::Type) -> Self::Value; + fn const_bitcast(&self, val: Self::Value, ty: Self::Type) -> Self::Value; + fn const_ptr_byte_offset(&self, val: Self::Value, offset: abi::Size) -> Self::Value; } diff --git a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs index e77201cf0..7e8de0ddc 100644 --- a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs @@ -1,57 +1,11 @@ use super::BackendTypes; -use rustc_hir::def_id::DefId; -use rustc_middle::mir::coverage::*; +use rustc_middle::mir::Coverage; use rustc_middle::ty::Instance; -pub trait CoverageInfoMethods<'tcx>: BackendTypes { - fn coverageinfo_finalize(&self); - - /// Codegen a small function that will never be called, with one counter - /// that will never be incremented, that gives LLVM coverage tools a - /// function definition it needs in order to resolve coverage map references - /// to unused functions. This is necessary so unused functions will appear - /// as uncovered (coverage execution count `0`) in LLVM coverage reports. - fn define_unused_fn(&self, def_id: DefId); - - /// For LLVM codegen, returns a function-specific `Value` for a global - /// string, to hold the function name passed to LLVM intrinsic - /// `instrprof.increment()`. The `Value` is only created once per instance. - /// Multiple invocations with the same instance return the same `Value`. - fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> Self::Value; -} - pub trait CoverageInfoBuilderMethods<'tcx>: BackendTypes { - /// Returns true if the function source hash was added to the coverage map (even if it had - /// already been added, for this instance). Returns false *only* if `-C instrument-coverage` is - /// not enabled (a coverage map is not being generated). - fn set_function_source_hash( - &mut self, - instance: Instance<'tcx>, - function_source_hash: u64, - ) -> bool; - - /// Returns true if the counter was added to the coverage map; false if `-C instrument-coverage` - /// is not enabled (a coverage map is not being generated). - fn add_coverage_counter( - &mut self, - instance: Instance<'tcx>, - index: CounterValueReference, - region: CodeRegion, - ) -> bool; - - /// Returns true if the expression was added to the coverage map; false if - /// `-C instrument-coverage` is not enabled (a coverage map is not being generated). - fn add_coverage_counter_expression( - &mut self, - instance: Instance<'tcx>, - id: InjectedExpressionId, - lhs: ExpressionOperandId, - op: Op, - rhs: ExpressionOperandId, - region: Option<CodeRegion>, - ) -> bool; - - /// Returns true if the region was added to the coverage map; false if `-C instrument-coverage` - /// is not enabled (a coverage map is not being generated). - fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool; + /// Handle the MIR coverage info in a backend-specific way. + /// + /// This can potentially be a no-op in backends that don't support + /// coverage instrumentation. + fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage); } diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs index 782fdadbf..8cb58bd4c 100644 --- a/compiler/rustc_codegen_ssa/src/traits/mod.rs +++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs @@ -33,7 +33,7 @@ pub use self::asm::{AsmBuilderMethods, AsmMethods, GlobalAsmOperandRef, InlineAs pub use self::backend::{Backend, BackendTypes, CodegenBackend, ExtraBackendMethods}; pub use self::builder::{BuilderMethods, OverflowOp}; pub use self::consts::ConstMethods; -pub use self::coverageinfo::{CoverageInfoBuilderMethods, CoverageInfoMethods}; +pub use self::coverageinfo::CoverageInfoBuilderMethods; pub use self::debuginfo::{DebugInfoBuilderMethods, DebugInfoMethods}; pub use self::declare::PreDefineMethods; pub use self::intrinsic::IntrinsicCallMethods; @@ -59,7 +59,6 @@ pub trait CodegenMethods<'tcx>: + MiscMethods<'tcx> + ConstMethods<'tcx> + StaticMethods - + CoverageInfoMethods<'tcx> + DebugInfoMethods<'tcx> + AsmMethods<'tcx> + PreDefineMethods<'tcx> @@ -75,7 +74,6 @@ impl<'tcx, T> CodegenMethods<'tcx> for T where + MiscMethods<'tcx> + ConstMethods<'tcx> + StaticMethods - + CoverageInfoMethods<'tcx> + DebugInfoMethods<'tcx> + AsmMethods<'tcx> + PreDefineMethods<'tcx> diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 36d986422..e64417e1a 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -126,6 +126,28 @@ pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> { index: usize, immediate: bool, ) -> Self::Type; + + /// A type that can be used in a [`super::BuilderMethods::load`] + + /// [`super::BuilderMethods::store`] pair to implement a *typed* copy, + /// such as a MIR `*_0 = *_1`. + /// + /// It's always legal to return `None` here, as the provided impl does, + /// in which case callers should use [`super::BuilderMethods::memcpy`] + /// instead of the `load`+`store` pair. + /// + /// This can be helpful for things like arrays, where the LLVM backend type + /// `[3 x i16]` optimizes to three separate loads and stores, but it can + /// instead be copied via an `i48` that stays as the single `load`+`store`. + /// (As of 2023-05 LLVM cannot necessarily optimize away a `memcpy` in these + /// cases, due to `poison` handling, but in codegen we have more information + /// about the type invariants, so can emit something better instead.) + /// + /// This *should* return `None` for particularly-large types, where leaving + /// the `memcpy` may well be important to avoid code size explosion. + fn scalar_copy_backend_type(&self, layout: TyAndLayout<'tcx>) -> Option<Self::Type> { + let _ = layout; + None + } } // For backends that support CFI using type membership (i.e., testing whether a given pointer is |