summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_codegen_ssa/src
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_codegen_ssa/src')
-rw-r--r--compiler/rustc_codegen_ssa/src/back/archive.rs73
-rw-r--r--compiler/rustc_codegen_ssa/src/back/link.rs402
-rw-r--r--compiler/rustc_codegen_ssa/src/back/linker.rs67
-rw-r--r--compiler/rustc_codegen_ssa/src/back/metadata.rs16
-rw-r--r--compiler/rustc_codegen_ssa/src/back/symbol_export.rs20
-rw-r--r--compiler/rustc_codegen_ssa/src/back/write.rs29
-rw-r--r--compiler/rustc_codegen_ssa/src/base.rs81
-rw-r--r--compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs56
-rw-r--r--compiler/rustc_codegen_ssa/src/lib.rs32
-rw-r--r--compiler/rustc_codegen_ssa/src/meth.rs21
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/analyze.rs6
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/block.rs221
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/constant.rs36
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/debuginfo.rs8
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/intrinsic.rs16
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/mod.rs14
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/operand.rs9
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/place.rs19
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/rvalue.rs18
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/statement.rs14
-rw-r--r--compiler/rustc_codegen_ssa/src/target_features.rs11
-rw-r--r--compiler/rustc_codegen_ssa/src/traits/builder.rs157
-rw-r--r--compiler/rustc_codegen_ssa/src/traits/consts.rs1
23 files changed, 646 insertions, 681 deletions
diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs
index 0d2aa483d..bb76ca5d2 100644
--- a/compiler/rustc_codegen_ssa/src/back/archive.rs
+++ b/compiler/rustc_codegen_ssa/src/back/archive.rs
@@ -1,44 +1,16 @@
+use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::memmap::Mmap;
use rustc_session::cstore::DllImport;
use rustc_session::Session;
+use rustc_span::symbol::Symbol;
+use object::read::archive::ArchiveFile;
+
+use std::fmt::Display;
+use std::fs::File;
use std::io;
use std::path::{Path, PathBuf};
-pub(super) fn find_library(
- name: &str,
- verbatim: bool,
- search_paths: &[PathBuf],
- sess: &Session,
-) -> PathBuf {
- // On Windows, static libraries sometimes show up as libfoo.a and other
- // times show up as foo.lib
- let oslibname = if verbatim {
- name.to_string()
- } else {
- format!("{}{}{}", sess.target.staticlib_prefix, name, sess.target.staticlib_suffix)
- };
- let unixlibname = format!("lib{}.a", name);
-
- for path in search_paths {
- debug!("looking for {} inside {:?}", name, path);
- let test = path.join(&oslibname);
- if test.exists() {
- return test;
- }
- if oslibname != unixlibname {
- let test = path.join(&unixlibname);
- if test.exists() {
- return test;
- }
- }
- }
- sess.fatal(&format!(
- "could not find native static library `{}`, \
- perhaps an -L flag is missing?",
- name
- ));
-}
-
pub trait ArchiveBuilderBuilder {
fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a>;
@@ -53,7 +25,38 @@ pub trait ArchiveBuilderBuilder {
lib_name: &str,
dll_imports: &[DllImport],
tmpdir: &Path,
+ is_direct_dependency: bool,
) -> PathBuf;
+
+ fn extract_bundled_libs(
+ &self,
+ rlib: &Path,
+ outdir: &Path,
+ bundled_lib_file_names: &FxHashSet<Symbol>,
+ ) -> Result<(), String> {
+ let message = |msg: &str, e: &dyn Display| format!("{} '{}': {}", msg, &rlib.display(), e);
+ let archive_map = unsafe {
+ Mmap::map(File::open(rlib).map_err(|e| message("failed to open file", &e))?)
+ .map_err(|e| message("failed to mmap file", &e))?
+ };
+ let archive = ArchiveFile::parse(&*archive_map)
+ .map_err(|e| message("failed to parse archive", &e))?;
+
+ for entry in archive.members() {
+ let entry = entry.map_err(|e| message("failed to read entry", &e))?;
+ let data = entry
+ .data(&*archive_map)
+ .map_err(|e| message("failed to get data from archive member", &e))?;
+ let name = std::str::from_utf8(entry.name())
+ .map_err(|e| message("failed to convert name", &e))?;
+ if !bundled_lib_file_names.contains(&Symbol::intern(name)) {
+ continue; // We need to extract only native libraries.
+ }
+ std::fs::write(&outdir.join(&name), data)
+ .map_err(|e| message("failed to write file", &e))?;
+ }
+ Ok(())
+ }
}
pub trait ArchiveBuilder<'a> {
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 63207803e..6cce95427 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -1,11 +1,13 @@
use rustc_arena::TypedArena;
use rustc_ast::CRATE_NODE_ID;
-use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
+use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexMap;
use rustc_data_structures::memmap::Mmap;
use rustc_data_structures::temp_dir::MaybeTempDir;
use rustc_errors::{ErrorGuaranteed, Handler};
use rustc_fs_util::fix_windows_verbatim_for_gcc;
use rustc_hir::def_id::CrateNum;
+use rustc_metadata::find_native_static_library;
use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME};
use rustc_middle::middle::dependency_format::Linkage;
use rustc_middle::middle::exported_symbols::SymbolExportKind;
@@ -20,11 +22,11 @@ use rustc_session::utils::NativeLibKind;
use rustc_session::{filesearch, Session};
use rustc_span::symbol::Symbol;
use rustc_span::DebuggerVisualizerFile;
-use rustc_target::spec::crt_objects::{CrtObjects, CrtObjectsFallback};
+use rustc_target::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault};
use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo};
use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Target};
-use super::archive::{find_library, ArchiveBuilder, ArchiveBuilderBuilder};
+use super::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
use super::command::Command;
use super::linker::{self, Linker};
use super::metadata::{create_rmeta_file, MetadataPosition};
@@ -44,7 +46,7 @@ use std::io::{BufWriter, Write};
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::process::{ExitStatus, Output, Stdio};
-use std::{ascii, char, env, fmt, fs, io, mem, str};
+use std::{env, fmt, fs, io, mem, str};
pub fn ensure_removed(diag_handler: &Handler, path: &Path) {
if let Err(e) = fs::remove_file(path) {
@@ -307,6 +309,9 @@ fn link_rlib<'a>(
}
}
+ // Used if packed_bundled_libs flag enabled.
+ let mut packed_bundled_libs = Vec::new();
+
// Note that in this loop we are ignoring the value of `lib.cfg`. That is,
// we may not be configured to actually include a static library if we're
// adding it here. That's because later when we consume this rlib we'll
@@ -326,6 +331,8 @@ fn link_rlib<'a>(
for lib in codegen_results.crate_info.used_libraries.iter() {
match lib.kind {
NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
+ if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.packed_bundled_libs => {}
+ NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
if flavor == RlibFlavor::Normal =>
{
// Don't allow mixing +bundle with +whole_archive since an rlib may contain
@@ -348,7 +355,16 @@ fn link_rlib<'a>(
}
if let Some(name) = lib.name {
let location =
- find_library(name.as_str(), lib.verbatim.unwrap_or(false), &lib_search_paths, sess);
+ find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
+ if sess.opts.unstable_opts.packed_bundled_libs && flavor == RlibFlavor::Normal {
+ packed_bundled_libs.push(find_native_static_library(
+ lib.filename.unwrap().as_str(),
+ Some(true),
+ &lib_search_paths,
+ sess,
+ ));
+ continue;
+ }
ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|e| {
sess.fatal(&format!(
"failed to add native library {}: {}",
@@ -360,13 +376,14 @@ fn link_rlib<'a>(
}
for (raw_dylib_name, raw_dylib_imports) in
- collate_raw_dylibs(sess, &codegen_results.crate_info.used_libraries)?
+ collate_raw_dylibs(sess, codegen_results.crate_info.used_libraries.iter())?
{
let output_path = archive_builder_builder.create_dll_import_lib(
sess,
&raw_dylib_name,
&raw_dylib_imports,
tmpdir.as_ref(),
+ true,
);
ab.add_archive(&output_path, Box::new(|_| false)).unwrap_or_else(|e| {
@@ -403,6 +420,12 @@ fn link_rlib<'a>(
ab.add_file(&trailing_metadata);
}
+ // Add all bundled static native library dependencies.
+ // Archives added to the end of .rlib archive, see comment above for the reason.
+ for lib in packed_bundled_libs {
+ ab.add_file(&lib)
+ }
+
return Ok(ab);
}
@@ -412,9 +435,9 @@ fn link_rlib<'a>(
/// then the CodegenResults value contains one NativeLib instance for each block. However, the
/// linker appears to expect only a single import library for each library used, so we need to
/// collate the symbols together by library name before generating the import libraries.
-fn collate_raw_dylibs(
- sess: &Session,
- used_libraries: &[NativeLib],
+fn collate_raw_dylibs<'a, 'b>(
+ sess: &'a Session,
+ used_libraries: impl IntoIterator<Item = &'b NativeLib>,
) -> Result<Vec<(String, Vec<DllImport>)>, ErrorGuaranteed> {
// Use index maps to preserve original order of imports and libraries.
let mut dylib_table = FxIndexMap::<String, FxIndexMap<Symbol, &DllImport>>::default();
@@ -552,14 +575,6 @@ fn link_staticlib<'a>(
Ok(())
}
-fn escape_stdout_stderr_string(s: &[u8]) -> String {
- str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| {
- let mut x = "Non-UTF-8 output: ".to_string();
- x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from));
- x
- })
-}
-
/// Use `thorin` (rust implementation of a dwarf packaging utility) to link DWARF objects into a
/// DWARF package.
fn link_dwarf_object<'a>(
@@ -764,15 +779,15 @@ fn link_natively<'a>(
"Linker does not support -static-pie command line option. Retrying with -static instead."
);
// Mirror `add_(pre,post)_link_objects` to replace CRT objects.
- let self_contained = crt_objects_fallback(sess, crate_type);
+ let self_contained = self_contained(sess, crate_type);
let opts = &sess.target;
let pre_objects = if self_contained {
- &opts.pre_link_objects_fallback
+ &opts.pre_link_objects_self_contained
} else {
&opts.pre_link_objects
};
let post_objects = if self_contained {
- &opts.post_link_objects_fallback
+ &opts.post_link_objects_self_contained
} else {
&opts.post_link_objects
};
@@ -866,7 +881,7 @@ fn link_natively<'a>(
if !prog.status.success() {
let mut output = prog.stderr.clone();
output.extend_from_slice(&prog.stdout);
- let escaped_output = escape_stdout_stderr_string(&output);
+ let escaped_output = escape_string(&output);
let mut err = sess.struct_err(&format!(
"linking with `{}` failed: {}",
linker_path.display(),
@@ -934,8 +949,8 @@ fn link_natively<'a>(
sess.abort_if_errors();
}
- info!("linker stderr:\n{}", escape_stdout_stderr_string(&prog.stderr));
- info!("linker stdout:\n{}", escape_stdout_stderr_string(&prog.stdout));
+ info!("linker stderr:\n{}", escape_string(&prog.stderr));
+ info!("linker stdout:\n{}", escape_string(&prog.stdout));
}
Err(e) => {
let linker_not_found = e.kind() == io::ErrorKind::NotFound;
@@ -1065,11 +1080,10 @@ fn strip_symbols_in_osx<'a>(sess: &'a Session, out_filename: &Path, option: Opti
}
fn escape_string(s: &[u8]) -> String {
- str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| {
- let mut x = "Non-UTF-8 output: ".to_string();
- x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from));
- x
- })
+ match str::from_utf8(s) {
+ Ok(s) => s.to_owned(),
+ Err(_) => format!("Non-UTF-8 output: {}", s.escape_ascii()),
+ }
}
fn add_sanitizer_libraries(sess: &Session, crate_type: CrateType, linker: &mut dyn Linker) {
@@ -1173,13 +1187,6 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
// only the linker flavor is known; use the default linker for the selected flavor
(None, Some(flavor)) => Some((
PathBuf::from(match flavor {
- LinkerFlavor::Em => {
- if cfg!(windows) {
- "emcc.bat"
- } else {
- "emcc"
- }
- }
LinkerFlavor::Gcc => {
if cfg!(any(target_os = "solaris", target_os = "illumos")) {
// On historical Solaris systems, "cc" may have
@@ -1194,11 +1201,17 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
}
}
LinkerFlavor::Ld => "ld",
- LinkerFlavor::Msvc => "link.exe",
LinkerFlavor::Lld(_) => "lld",
- LinkerFlavor::PtxLinker => "rust-ptx-linker",
- LinkerFlavor::BpfLinker => "bpf-linker",
- LinkerFlavor::L4Bender => "l4-bender",
+ LinkerFlavor::Msvc => "link.exe",
+ LinkerFlavor::EmCc => {
+ if cfg!(windows) {
+ "emcc.bat"
+ } else {
+ "emcc"
+ }
+ }
+ LinkerFlavor::Bpf => "bpf-linker",
+ LinkerFlavor::Ptx => "rust-ptx-linker",
}),
flavor,
)),
@@ -1208,7 +1221,7 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
});
let flavor = if stem == "emcc" {
- LinkerFlavor::Em
+ LinkerFlavor::EmCc
} else if stem == "gcc"
|| stem.ends_with("-gcc")
|| stem == "clang"
@@ -1236,7 +1249,8 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
// linker and linker flavor specified via command line have precedence over what the target
// specification specifies
- if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) {
+ let linker_flavor = sess.opts.cg.linker_flavor.map(LinkerFlavor::from_cli);
+ if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), linker_flavor) {
return ret;
}
@@ -1556,26 +1570,26 @@ fn detect_self_contained_mingw(sess: &Session) -> bool {
true
}
-/// Whether we link to our own CRT objects instead of relying on gcc to pull them.
+/// Various toolchain components used during linking are used from rustc distribution
+/// instead of being found somewhere on the host system.
/// We only provide such support for a very limited number of targets.
-fn crt_objects_fallback(sess: &Session, crate_type: CrateType) -> bool {
+fn self_contained(sess: &Session, crate_type: CrateType) -> bool {
if let Some(self_contained) = sess.opts.cg.link_self_contained {
return self_contained;
}
- match sess.target.crt_objects_fallback {
+ match sess.target.link_self_contained {
+ LinkSelfContainedDefault::False => false,
+ LinkSelfContainedDefault::True => true,
// FIXME: Find a better heuristic for "native musl toolchain is available",
// based on host and linker path, for example.
// (https://github.com/rust-lang/rust/pull/71769#issuecomment-626330237).
- Some(CrtObjectsFallback::Musl) => sess.crt_static(Some(crate_type)),
- Some(CrtObjectsFallback::Mingw) => {
+ LinkSelfContainedDefault::Musl => sess.crt_static(Some(crate_type)),
+ LinkSelfContainedDefault::Mingw => {
sess.host == sess.target
&& sess.target.vendor != "uwp"
&& detect_self_contained_mingw(&sess)
}
- // FIXME: Figure out cases in which WASM needs to link with a native toolchain.
- Some(CrtObjectsFallback::Wasm) => true,
- None => false,
}
}
@@ -1583,12 +1597,21 @@ fn crt_objects_fallback(sess: &Session, crate_type: CrateType) -> bool {
fn add_pre_link_objects(
cmd: &mut dyn Linker,
sess: &Session,
+ flavor: LinkerFlavor,
link_output_kind: LinkOutputKind,
self_contained: bool,
) {
+ // FIXME: we are currently missing some infra here (per-linker-flavor CRT objects),
+ // so Fuchsia has to be special-cased.
let opts = &sess.target;
- let objects =
- if self_contained { &opts.pre_link_objects_fallback } else { &opts.pre_link_objects };
+ let empty = Default::default();
+ let objects = if self_contained {
+ &opts.pre_link_objects_self_contained
+ } else if !(sess.target.os == "fuchsia" && flavor == LinkerFlavor::Gcc) {
+ &opts.pre_link_objects
+ } else {
+ &empty
+ };
for obj in objects.get(&link_output_kind).iter().copied().flatten() {
cmd.add_object(&get_object_file_path(sess, obj, self_contained));
}
@@ -1601,9 +1624,11 @@ fn add_post_link_objects(
link_output_kind: LinkOutputKind,
self_contained: bool,
) {
- let opts = &sess.target;
- let objects =
- if self_contained { &opts.post_link_objects_fallback } else { &opts.post_link_objects };
+ let objects = if self_contained {
+ &sess.target.post_link_objects_self_contained
+ } else {
+ &sess.target.post_link_objects
+ };
for obj in objects.get(&link_output_kind).iter().copied().flatten() {
cmd.add_object(&get_object_file_path(sess, obj, self_contained));
}
@@ -1703,6 +1728,13 @@ fn add_post_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor
/// that are necessary for the linking. They are only present in symbol table but not actually
/// used in any sections, so the linker will therefore pick relevant rlibs for linking, but
/// unused `#[no_mangle]` or `#[used]` can still be discard by GC sections.
+///
+/// There's a few internal crates in the standard library (aka libcore and
+/// libstd) which actually have a circular dependence upon one another. This
+/// currently arises through "weak lang items" where libcore requires things
+/// like `rust_begin_unwind` but libstd ends up defining it. To get this
+/// circular dependence to work correctly we declare some of these things
+/// in this synthetic object.
fn add_linked_symbol_object(
cmd: &mut dyn Linker,
sess: &Session,
@@ -1882,12 +1914,12 @@ fn linker_with_args<'a>(
out_filename: &Path,
codegen_results: &CodegenResults,
) -> Result<Command, ErrorGuaranteed> {
- let crt_objects_fallback = crt_objects_fallback(sess, crate_type);
+ let self_contained = self_contained(sess, crate_type);
let cmd = &mut *super::linker::get_linker(
sess,
path,
flavor,
- crt_objects_fallback,
+ self_contained,
&codegen_results.crate_info.target_cpu,
);
let link_output_kind = link_output_kind(sess, crate_type);
@@ -1914,7 +1946,7 @@ fn linker_with_args<'a>(
// ------------ Object code and libraries, order-dependent ------------
// Pre-link CRT objects.
- add_pre_link_objects(cmd, sess, link_output_kind, crt_objects_fallback);
+ add_pre_link_objects(cmd, sess, flavor, link_output_kind, self_contained);
add_linked_symbol_object(
cmd,
@@ -1947,7 +1979,6 @@ fn linker_with_args<'a>(
// Upstream rust libraries are not supposed to depend on our local native
// libraries as that would violate the structure of the DAG, in that
// scenario they are required to link to them as well in a shared fashion.
- // (The current implementation still doesn't prevent it though, see the FIXME below.)
//
// Note that upstream rust libraries may contain native dependencies as
// well, but they also can't depend on what we just started to add to the
@@ -1968,15 +1999,16 @@ fn linker_with_args<'a>(
// and move this option back to the top.
cmd.add_as_needed();
- // FIXME: Move this below to other native libraries
- // (or alternatively link all native libraries after their respective crates).
- // This change is somewhat breaking in practice due to local static libraries being linked
- // as whole-archive (#85144), so removing whole-archive may be a pre-requisite.
+ // Local native libraries of all kinds.
+ //
+ // If `-Zlink-native-libraries=false` is set, then the assumption is that an
+ // external build system already has the native dependencies defined, and it
+ // will provide them to the linker itself.
if sess.opts.unstable_opts.link_native_libraries {
add_local_native_libraries(cmd, sess, codegen_results);
}
- // Upstream rust libraries and their non-bundled static libraries
+ // Upstream rust libraries and their (possibly bundled) static native libraries.
add_upstream_rust_crates(
cmd,
sess,
@@ -1986,24 +2018,54 @@ fn linker_with_args<'a>(
tmpdir,
);
- // Upstream dynamic native libraries linked with `#[link]` attributes at and `-l`
- // command line options.
- // If -Zlink-native-libraries=false is set, then the assumption is that an
- // external build system already has the native dependencies defined, and it
- // will provide them to the linker itself.
+ // Dynamic native libraries from upstream crates.
+ //
+ // FIXME: Merge this to `add_upstream_rust_crates` so that all native libraries are linked
+ // together with their respective upstream crates, and in their originally specified order.
+ // This may be slightly breaking due to our use of `--as-needed` and needs a crater run.
if sess.opts.unstable_opts.link_native_libraries {
add_upstream_native_libraries(cmd, sess, codegen_results);
}
// Link with the import library generated for any raw-dylib functions.
for (raw_dylib_name, raw_dylib_imports) in
- collate_raw_dylibs(sess, &codegen_results.crate_info.used_libraries)?
+ collate_raw_dylibs(sess, codegen_results.crate_info.used_libraries.iter())?
{
cmd.add_object(&archive_builder_builder.create_dll_import_lib(
sess,
&raw_dylib_name,
&raw_dylib_imports,
tmpdir,
+ true,
+ ));
+ }
+ // As with add_upstream_native_libraries, we need to add the upstream raw-dylib symbols in case
+ // they are used within inlined functions or instantiated generic functions. We do this *after*
+ // handling the raw-dylib symbols in the current crate to make sure that those are chosen first
+ // by the linker.
+ let (_, dependency_linkage) = codegen_results
+ .crate_info
+ .dependency_formats
+ .iter()
+ .find(|(ty, _)| *ty == crate_type)
+ .expect("failed to find crate type in dependency format list");
+ let native_libraries_from_nonstatics = codegen_results
+ .crate_info
+ .native_libraries
+ .iter()
+ .filter_map(|(cnum, libraries)| {
+ (dependency_linkage[cnum.as_usize() - 1] != Linkage::Static).then(|| libraries)
+ })
+ .flatten();
+ for (raw_dylib_name, raw_dylib_imports) in
+ collate_raw_dylibs(sess, native_libraries_from_nonstatics)?
+ {
+ cmd.add_object(&archive_builder_builder.create_dll_import_lib(
+ sess,
+ &raw_dylib_name,
+ &raw_dylib_imports,
+ tmpdir,
+ false,
));
}
@@ -2024,7 +2086,7 @@ fn linker_with_args<'a>(
cmd,
sess,
link_output_kind,
- crt_objects_fallback,
+ self_contained,
flavor,
crate_type,
codegen_results,
@@ -2040,7 +2102,7 @@ fn linker_with_args<'a>(
// ------------ Object code and libraries, order-dependent ------------
// Post-link CRT objects.
- add_post_link_objects(cmd, sess, link_output_kind, crt_objects_fallback);
+ add_post_link_objects(cmd, sess, link_output_kind, self_contained);
// ------------ Late order-dependent options ------------
@@ -2057,7 +2119,7 @@ fn add_order_independent_options(
cmd: &mut dyn Linker,
sess: &Session,
link_output_kind: LinkOutputKind,
- crt_objects_fallback: bool,
+ self_contained: bool,
flavor: LinkerFlavor,
crate_type: CrateType,
codegen_results: &CodegenResults,
@@ -2070,7 +2132,10 @@ fn add_order_independent_options(
add_link_script(cmd, sess, tmpdir, crate_type);
- if sess.target.os == "fuchsia" && crate_type == CrateType::Executable {
+ if sess.target.os == "fuchsia"
+ && crate_type == CrateType::Executable
+ && flavor != LinkerFlavor::Gcc
+ {
let prefix = if sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::ADDRESS) {
"asan/"
} else {
@@ -2086,7 +2151,7 @@ fn add_order_independent_options(
// Make the binary compatible with data execution prevention schemes.
cmd.add_no_exec();
- if crt_objects_fallback {
+ if self_contained {
cmd.no_crt_objects();
}
@@ -2099,11 +2164,11 @@ fn add_order_independent_options(
});
}
- if flavor == LinkerFlavor::PtxLinker {
+ if flavor == LinkerFlavor::Ptx {
// Provide the linker with fallback to internal `target-cpu`.
cmd.arg("--fallback-arch");
cmd.arg(&codegen_results.crate_info.target_cpu);
- } else if flavor == LinkerFlavor::BpfLinker {
+ } else if flavor == LinkerFlavor::Bpf {
cmd.arg("--cpu");
cmd.arg(&codegen_results.crate_info.target_cpu);
cmd.arg("--cpu-features");
@@ -2115,7 +2180,7 @@ fn add_order_independent_options(
cmd.linker_plugin_lto();
- add_library_search_dirs(cmd, sess, crt_objects_fallback);
+ add_library_search_dirs(cmd, sess, self_contained);
cmd.output_filename(out_filename);
@@ -2319,72 +2384,25 @@ fn add_upstream_rust_crates<'a>(
// crates.
let deps = &codegen_results.crate_info.used_crates;
- // There's a few internal crates in the standard library (aka libcore and
- // libstd) which actually have a circular dependence upon one another. This
- // currently arises through "weak lang items" where libcore requires things
- // like `rust_begin_unwind` but libstd ends up defining it. To get this
- // circular dependence to work correctly in all situations we'll need to be
- // sure to correctly apply the `--start-group` and `--end-group` options to
- // GNU linkers, otherwise if we don't use any other symbol from the standard
- // library it'll get discarded and the whole application won't link.
- //
- // In this loop we're calculating the `group_end`, after which crate to
- // pass `--end-group` and `group_start`, before which crate to pass
- // `--start-group`. We currently do this by passing `--end-group` after
- // the first crate (when iterating backwards) that requires a lang item
- // defined somewhere else. Once that's set then when we've defined all the
- // necessary lang items we'll pass `--start-group`.
- //
- // Note that this isn't amazing logic for now but it should do the trick
- // for the current implementation of the standard library.
- let mut group_end = None;
- let mut group_start = None;
- // Crates available for linking thus far.
- let mut available = FxHashSet::default();
- // Crates required to satisfy dependencies discovered so far.
- let mut required = FxHashSet::default();
-
- let info = &codegen_results.crate_info;
- for &cnum in deps.iter().rev() {
- if let Some(missing) = info.missing_lang_items.get(&cnum) {
- let missing_crates = missing.iter().map(|i| info.lang_item_to_crate.get(i).copied());
- required.extend(missing_crates);
- }
-
- required.insert(Some(cnum));
- available.insert(Some(cnum));
-
- if required.len() > available.len() && group_end.is_none() {
- group_end = Some(cnum);
- }
- if required.len() == available.len() && group_end.is_some() {
- group_start = Some(cnum);
- break;
- }
- }
-
- // If we didn't end up filling in all lang items from upstream crates then
- // we'll be filling it in with our crate. This probably means we're the
- // standard library itself, so skip this for now.
- if group_end.is_some() && group_start.is_none() {
- group_end = None;
- }
-
let mut compiler_builtins = None;
let search_path = OnceCell::new();
for &cnum in deps.iter() {
- if group_start == Some(cnum) {
- cmd.group_start();
- }
-
// We may not pass all crates through to the linker. Some crates may
// appear statically in an existing dylib, meaning we'll pick up all the
// symbols from the dylib.
let src = &codegen_results.crate_info.used_crate_source[&cnum];
match data[cnum.as_usize() - 1] {
_ if codegen_results.crate_info.profiler_runtime == Some(cnum) => {
- add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
+ add_static_crate(
+ cmd,
+ sess,
+ archive_builder_builder,
+ codegen_results,
+ tmpdir,
+ cnum,
+ &Default::default(),
+ );
}
// compiler-builtins are always placed last to ensure that they're
// linked correctly.
@@ -2394,17 +2412,41 @@ fn add_upstream_rust_crates<'a>(
}
Linkage::NotLinked | Linkage::IncludedFromDylib => {}
Linkage::Static => {
- add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
+ let bundled_libs = if sess.opts.unstable_opts.packed_bundled_libs {
+ codegen_results.crate_info.native_libraries[&cnum]
+ .iter()
+ .filter_map(|lib| lib.filename)
+ .collect::<FxHashSet<_>>()
+ } else {
+ Default::default()
+ };
+ add_static_crate(
+ cmd,
+ sess,
+ archive_builder_builder,
+ codegen_results,
+ tmpdir,
+ cnum,
+ &bundled_libs,
+ );
// Link static native libs with "-bundle" modifier only if the crate they originate from
// is being linked statically to the current crate. If it's linked dynamically
// or is an rlib already included via some other dylib crate, the symbols from
// native libs will have already been included in that dylib.
//
- // If -Zlink-native-libraries=false is set, then the assumption is that an
+ // If `-Zlink-native-libraries=false` is set, then the assumption is that an
// external build system already has the native dependencies defined, and it
// will provide them to the linker itself.
if sess.opts.unstable_opts.link_native_libraries {
+ if sess.opts.unstable_opts.packed_bundled_libs {
+ // If rlib contains native libs as archives, unpack them to tmpdir.
+ let rlib = &src.rlib.as_ref().unwrap().0;
+ archive_builder_builder
+ .extract_bundled_libs(rlib, tmpdir, &bundled_libs)
+ .unwrap_or_else(|e| sess.fatal(e));
+ }
+
let mut last = (None, NativeLibKind::Unspecified, None);
for lib in &codegen_results.crate_info.native_libraries[&cnum] {
let Some(name) = lib.name else {
@@ -2437,6 +2479,14 @@ fn add_upstream_rust_crates<'a>(
bundle: Some(false),
whole_archive: Some(false) | None,
} => {
+ // HACK/FIXME: Fixup a circular dependency between libgcc and libc
+ // with glibc. This logic should be moved to the libc crate.
+ if sess.target.os == "linux"
+ && sess.target.env == "gnu"
+ && name == "c"
+ {
+ cmd.link_staticlib("gcc", false);
+ }
cmd.link_staticlib(name, lib.verbatim.unwrap_or(false));
}
NativeLibKind::LinkArg => {
@@ -2446,20 +2496,23 @@ fn add_upstream_rust_crates<'a>(
| NativeLibKind::Framework { .. }
| NativeLibKind::Unspecified
| NativeLibKind::RawDylib => {}
- NativeLibKind::Static {
- bundle: Some(true) | None,
- whole_archive: _,
- } => {}
+ NativeLibKind::Static { bundle: Some(true) | None, whole_archive } => {
+ if sess.opts.unstable_opts.packed_bundled_libs {
+ // If rlib contains native libs as archives, they are unpacked to tmpdir.
+ let path = tmpdir.join(lib.filename.unwrap().as_str());
+ if whole_archive == Some(true) {
+ cmd.link_whole_rlib(&path);
+ } else {
+ cmd.link_rlib(&path);
+ }
+ }
+ }
}
}
}
}
Linkage::Dynamic => add_dynamic_crate(cmd, sess, &src.dylib.as_ref().unwrap().0),
}
-
- if group_end == Some(cnum) {
- cmd.group_end();
- }
}
// compiler-builtins are always placed last to ensure that they're
@@ -2468,7 +2521,15 @@ fn add_upstream_rust_crates<'a>(
// was already "included" in a dylib (e.g., `libstd` when `-C prefer-dynamic`
// is used)
if let Some(cnum) = compiler_builtins {
- add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
+ add_static_crate(
+ cmd,
+ sess,
+ archive_builder_builder,
+ codegen_results,
+ tmpdir,
+ cnum,
+ &Default::default(),
+ );
}
// Converts a library file-stem into a cc -l argument
@@ -2501,6 +2562,7 @@ fn add_upstream_rust_crates<'a>(
codegen_results: &CodegenResults,
tmpdir: &Path,
cnum: CrateNum,
+ bundled_lib_file_names: &FxHashSet<Symbol>,
) {
let src = &codegen_results.crate_info.used_crate_source[&cnum];
let cratepath = &src.rlib.as_ref().unwrap().0;
@@ -2529,6 +2591,7 @@ fn add_upstream_rust_crates<'a>(
let dst = tmpdir.join(cratepath.file_name().unwrap());
let name = cratepath.file_name().unwrap().to_str().unwrap();
let name = &name[3..name.len() - 5]; // chop off lib/.rlib
+ let bundled_lib_file_names = bundled_lib_file_names.clone();
sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| {
let canonical_name = name.replace('-', "_");
@@ -2562,6 +2625,15 @@ fn add_upstream_rust_crates<'a>(
let skip_because_lto =
upstream_rust_objects_already_included && is_rust_object && is_builtins;
+ // We skip native libraries because:
+ // 1. This native libraries won't be used from the generated rlib,
+ // so we can throw them away to avoid the copying work.
+ // 2. We can't allow it to be a single remaining entry in archive
+ // as some linkers may complain on that.
+ if bundled_lib_file_names.contains(&Symbol::intern(f)) {
+ return true;
+ }
+
if skip_because_cfg_say_so || skip_because_lto {
return true;
}
@@ -2657,7 +2729,7 @@ fn relevant_lib(sess: &Session, lib: &NativeLib) -> bool {
}
}
-fn are_upstream_rust_objects_already_included(sess: &Session) -> bool {
+pub(crate) fn are_upstream_rust_objects_already_included(sess: &Session) -> bool {
match sess.lto() {
config::Lto::Fat => true,
config::Lto::Thin => {
@@ -2674,11 +2746,16 @@ fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
let os = &sess.target.os;
let llvm_target = &sess.target.llvm_target;
if sess.target.vendor != "apple"
- || !matches!(os.as_ref(), "ios" | "tvos" | "watchos")
+ || !matches!(os.as_ref(), "ios" | "tvos" | "watchos" | "macos")
|| (flavor != LinkerFlavor::Gcc && flavor != LinkerFlavor::Lld(LldFlavor::Ld64))
{
return;
}
+
+ if os == "macos" && flavor != LinkerFlavor::Lld(LldFlavor::Ld64) {
+ return;
+ }
+
let sdk_name = match (arch.as_ref(), os.as_ref()) {
("aarch64", "tvos") => "appletvos",
("x86_64", "tvos") => "appletvsimulator",
@@ -2694,6 +2771,7 @@ fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
("aarch64", "watchos") if llvm_target.ends_with("-simulator") => "watchsimulator",
("aarch64", "watchos") => "watchos",
("arm", "watchos") => "watchos",
+ (_, "macos") => "macosx",
_ => {
sess.err(&format!("unsupported arch `{}` for os `{}`", arch, os));
return;
@@ -2777,20 +2855,24 @@ fn add_gcc_ld_path(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
if let LinkerFlavor::Gcc = flavor {
match ld_impl {
LdImpl::Lld => {
- let tools_path = sess.get_tools_search_paths(false);
- let gcc_ld_dir = tools_path
- .into_iter()
- .map(|p| p.join("gcc-ld"))
- .find(|p| {
- p.join(if sess.host.is_like_windows { "ld.exe" } else { "ld" }).exists()
- })
- .unwrap_or_else(|| sess.fatal("rust-lld (as ld) not found"));
- cmd.arg({
- let mut arg = OsString::from("-B");
- arg.push(gcc_ld_dir);
- arg
- });
- cmd.arg(format!("-Wl,-rustc-lld-flavor={}", sess.target.lld_flavor.as_str()));
+ // Implement the "self-contained" part of -Zgcc-ld
+ // by adding rustc distribution directories to the tool search path.
+ for path in sess.get_tools_search_paths(false) {
+ cmd.arg({
+ let mut arg = OsString::from("-B");
+ arg.push(path.join("gcc-ld"));
+ arg
+ });
+ }
+ // Implement the "linker flavor" part of -Zgcc-ld
+ // by asking cc to use some kind of lld.
+ cmd.arg("-fuse-ld=lld");
+ if sess.target.lld_flavor != LldFlavor::Ld {
+ // Tell clang to use a non-default LLD flavor.
+ // Gcc doesn't understand the target option, but we currently assume
+ // that gcc is not used for Apple and Wasm targets (#97402).
+ cmd.arg(format!("--target={}", sess.target.llvm_target));
+ }
}
}
} else {
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index ce51b2e95..e505543b2 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -1,4 +1,3 @@
-use super::archive;
use super::command::Command;
use super::symbol_export;
use rustc_span::symbol::sym;
@@ -11,6 +10,7 @@ use std::path::{Path, PathBuf};
use std::{env, mem, str};
use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
+use rustc_metadata::find_native_static_library;
use rustc_middle::middle::dependency_format::Linkage;
use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind};
use rustc_middle::ty::TyCtxt;
@@ -126,29 +126,26 @@ pub fn get_linker<'a>(
// to the linker args construction.
assert!(cmd.get_args().is_empty() || sess.target.vendor == "uwp");
match flavor {
- LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => {
- Box::new(MsvcLinker { cmd, sess }) as Box<dyn Linker>
- }
- LinkerFlavor::Em => Box::new(EmLinker { cmd, sess }) as Box<dyn Linker>,
LinkerFlavor::Gcc => {
Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: false })
as Box<dyn Linker>
}
-
+ LinkerFlavor::Ld if sess.target.os == "l4re" => {
+ Box::new(L4Bender::new(cmd, sess)) as Box<dyn Linker>
+ }
LinkerFlavor::Lld(LldFlavor::Ld)
| LinkerFlavor::Lld(LldFlavor::Ld64)
| LinkerFlavor::Ld => {
Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: true })
as Box<dyn Linker>
}
-
+ LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => {
+ Box::new(MsvcLinker { cmd, sess }) as Box<dyn Linker>
+ }
LinkerFlavor::Lld(LldFlavor::Wasm) => Box::new(WasmLd::new(cmd, sess)) as Box<dyn Linker>,
-
- LinkerFlavor::PtxLinker => Box::new(PtxLinker { cmd, sess }) as Box<dyn Linker>,
-
- LinkerFlavor::BpfLinker => Box::new(BpfLinker { cmd, sess }) as Box<dyn Linker>,
-
- LinkerFlavor::L4Bender => Box::new(L4Bender::new(cmd, sess)) as Box<dyn Linker>,
+ LinkerFlavor::EmCc => Box::new(EmLinker { cmd, sess }) as Box<dyn Linker>,
+ LinkerFlavor::Bpf => Box::new(BpfLinker { cmd, sess }) as Box<dyn Linker>,
+ LinkerFlavor::Ptx => Box::new(PtxLinker { cmd, sess }) as Box<dyn Linker>,
}
}
@@ -186,8 +183,6 @@ pub trait Linker {
fn no_default_libraries(&mut self);
fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]);
fn subsystem(&mut self, subsystem: &str);
- fn group_start(&mut self);
- fn group_end(&mut self);
fn linker_plugin_lto(&mut self);
fn add_eh_frame_header(&mut self) {}
fn add_no_exec(&mut self) {}
@@ -519,7 +514,7 @@ impl<'a> Linker for GccLinker<'a> {
// -force_load is the macOS equivalent of --whole-archive, but it
// involves passing the full path to the library to link.
self.linker_arg("-force_load");
- let lib = archive::find_library(lib, verbatim, search_path, &self.sess);
+ let lib = find_native_static_library(lib, Some(verbatim), search_path, &self.sess);
self.linker_arg(&lib);
}
}
@@ -733,18 +728,6 @@ impl<'a> Linker for GccLinker<'a> {
self.hint_dynamic(); // Reset to default before returning the composed command line.
}
- fn group_start(&mut self) {
- if self.takes_hints() {
- self.linker_arg("--start-group");
- }
- }
-
- fn group_end(&mut self) {
- if self.takes_hints() {
- self.linker_arg("--end-group");
- }
- }
-
fn linker_plugin_lto(&mut self) {
match self.sess.opts.cg.linker_plugin_lto {
LinkerPluginLto::Disabled => {
@@ -1022,10 +1005,6 @@ impl<'a> Linker for MsvcLinker<'a> {
}
}
- // MSVC doesn't need group indicators
- fn group_start(&mut self) {}
- fn group_end(&mut self) {}
-
fn linker_plugin_lto(&mut self) {
// Do nothing
}
@@ -1168,10 +1147,6 @@ impl<'a> Linker for EmLinker<'a> {
// noop
}
- // Appears not necessary on Emscripten
- fn group_start(&mut self) {}
- fn group_end(&mut self) {}
-
fn linker_plugin_lto(&mut self) {
// Do nothing
}
@@ -1347,10 +1322,6 @@ impl<'a> Linker for WasmLd<'a> {
fn subsystem(&mut self, _subsystem: &str) {}
- // Not needed for now with LLD
- fn group_start(&mut self) {}
- fn group_end(&mut self) {}
-
fn linker_plugin_lto(&mut self) {
// Do nothing for now
}
@@ -1479,14 +1450,6 @@ impl<'a> Linker for L4Bender<'a> {
self.hint_static(); // Reset to default before returning the composed command line.
}
- fn group_start(&mut self) {
- self.cmd.arg("--start-group");
- }
-
- fn group_end(&mut self) {
- self.cmd.arg("--end-group");
- }
-
fn linker_plugin_lto(&mut self) {}
fn control_flow_guard(&mut self) {}
@@ -1667,10 +1630,6 @@ impl<'a> Linker for PtxLinker<'a> {
fn subsystem(&mut self, _subsystem: &str) {}
- fn group_start(&mut self) {}
-
- fn group_end(&mut self) {}
-
fn linker_plugin_lto(&mut self) {}
}
@@ -1780,9 +1739,5 @@ impl<'a> Linker for BpfLinker<'a> {
fn subsystem(&mut self, _subsystem: &str) {}
- fn group_start(&mut self) {}
-
- fn group_end(&mut self) {}
-
fn linker_plugin_lto(&mut self) {}
}
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index 0302c2881..99ddd1764 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -117,6 +117,10 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
"riscv32" => Architecture::Riscv32,
"riscv64" => Architecture::Riscv64,
"sparc64" => Architecture::Sparc64,
+ "avr" => Architecture::Avr,
+ "msp430" => Architecture::Msp430,
+ "hexagon" => Architecture::Hexagon,
+ "bpf" => Architecture::Bpf,
// Unsupported architecture.
_ => return None,
};
@@ -187,12 +191,12 @@ pub enum MetadataPosition {
Last,
}
-// For rlibs we "pack" rustc metadata into a dummy object file. When rustc
-// creates a dylib crate type it will pass `--whole-archive` (or the
-// platform equivalent) to include all object files from an rlib into the
-// final dylib itself. This causes linkers to iterate and try to include all
-// files located in an archive, so if metadata is stored in an archive then
-// it needs to be of a form that the linker will be able to process.
+// For rlibs we "pack" rustc metadata into a dummy object file.
+//
+// Historically it was needed because rustc linked rlibs as whole-archive in some cases.
+// In that case linkers try to include all files located in an archive, so if metadata is stored
+// in an archive then it needs to be of a form that the linker is able to process.
+// Now it's not clear whether metadata still needs to be wrapped into an object file or not.
//
// Note, though, that we don't actually want this metadata to show up in any
// final output of the compiler. Instead this is purely for rustc's own
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index e6b605575..8d7e2c5cf 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -103,18 +103,14 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
}
})
.map(|def_id| {
- let (export_level, used) = if special_runtime_crate {
- let name = tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())).name;
- // We won't link right if these symbols are stripped during LTO.
- let used = match name {
- "rust_eh_personality"
- | "rust_eh_register_frames"
- | "rust_eh_unregister_frames" => true,
- _ => false,
- };
- (SymbolExportLevel::Rust, used)
+ // We won't link right if this symbol is stripped during LTO.
+ let name = tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())).name;
+ let used = name == "rust_eh_personality";
+
+ let export_level = if special_runtime_crate {
+ SymbolExportLevel::Rust
} else {
- (symbol_export_level(tcx, def_id.to_def_id()), false)
+ symbol_export_level(tcx, def_id.to_def_id())
};
let codegen_attrs = tcx.codegen_fn_attrs(def_id.to_def_id());
debug!(
@@ -544,7 +540,7 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
.map(|fnabi| (fnabi.conv, &fnabi.args[..]))
.unwrap_or((Conv::Rust, &[]));
- // Decorate symbols with prefices, suffices and total number of bytes of arguments.
+ // Decorate symbols with prefixes, suffixes and total number of bytes of arguments.
// Reference: https://docs.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170
let (prefix, suffix) = match conv {
Conv::X86Fastcall => ("@", "@"),
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 1b5ad8710..68f3b19b7 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -15,7 +15,7 @@ use rustc_data_structures::profiling::TimingGuard;
use rustc_data_structures::profiling::VerboseTimingGuard;
use rustc_data_structures::sync::Lrc;
use rustc_errors::emitter::Emitter;
-use rustc_errors::{DiagnosticId, FatalError, Handler, Level};
+use rustc_errors::{translation::Translate, DiagnosticId, FatalError, Handler, Level};
use rustc_fs_util::link_or_copy;
use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
use rustc_incremental::{
@@ -256,8 +256,11 @@ impl ModuleConfig {
{
MergeFunctions::Disabled => false,
MergeFunctions::Trampolines | MergeFunctions::Aliases => {
- sess.opts.optimize == config::OptLevel::Default
- || sess.opts.optimize == config::OptLevel::Aggressive
+ use config::OptLevel::*;
+ match sess.opts.optimize {
+ Aggressive | Default | SizeMin | Size => true,
+ Less | No => false,
+ }
}
},
@@ -1737,6 +1740,16 @@ impl SharedEmitter {
}
}
+impl Translate for SharedEmitter {
+ fn fluent_bundle(&self) -> Option<&Lrc<rustc_errors::FluentBundle>> {
+ None
+ }
+
+ fn fallback_fluent_bundle(&self) -> &rustc_errors::FluentBundle {
+ panic!("shared emitter attempted to translate a diagnostic");
+ }
+}
+
impl Emitter for SharedEmitter {
fn emit_diagnostic(&mut self, diag: &rustc_errors::Diagnostic) {
let fluent_args = self.to_fluent_args(diag.args());
@@ -1758,14 +1771,6 @@ impl Emitter for SharedEmitter {
fn source_map(&self) -> Option<&Lrc<SourceMap>> {
None
}
-
- fn fluent_bundle(&self) -> Option<&Lrc<rustc_errors::FluentBundle>> {
- None
- }
-
- fn fallback_fluent_bundle(&self) -> &rustc_errors::FluentBundle {
- panic!("shared emitter attempted to translate a diagnostic");
- }
}
impl SharedEmitterMain {
@@ -1887,7 +1892,7 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> {
}
});
- sess.cgu_reuse_tracker.check_expected_reuse(sess.diagnostic());
+ sess.cgu_reuse_tracker.check_expected_reuse(sess);
sess.abort_if_errors();
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index a840b2709..b98ff4957 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -1,3 +1,4 @@
+use crate::back::link::are_upstream_rust_objects_already_included;
use crate::back::metadata::create_compressed_metadata_file;
use crate::back::write::{
compute_per_cgu_lto_type, start_async_codegen, submit_codegened_module_to_llvm,
@@ -12,7 +13,7 @@ use crate::traits::*;
use crate::{CachedModuleCodegen, CompiledModule, CrateInfo, MemFlags, ModuleCodegen, ModuleKind};
use rustc_attr as attr;
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
use rustc_data_structures::sync::par_iter;
@@ -21,10 +22,12 @@ use rustc_data_structures::sync::ParallelIterator;
use rustc_hir as hir;
use rustc_hir::def_id::{DefId, LOCAL_CRATE};
use rustc_hir::lang_items::LangItem;
+use rustc_hir::weak_lang_items::WEAK_ITEMS_SYMBOLS;
use rustc_index::vec::Idx;
use rustc_metadata::EncodedMetadata;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
use rustc_middle::middle::exported_symbols;
+use rustc_middle::middle::exported_symbols::SymbolExportKind;
use rustc_middle::middle::lang_items;
use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem};
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
@@ -34,6 +37,7 @@ use rustc_session::cgu_reuse_tracker::CguReuse;
use rustc_session::config::{self, CrateType, EntryFnType, OutputType};
use rustc_session::Session;
use rustc_span::symbol::sym;
+use rustc_span::Symbol;
use rustc_span::{DebuggerVisualizerFile, DebuggerVisualizerType};
use rustc_target::abi::{Align, VariantIdx};
@@ -151,6 +155,7 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
let old_info =
old_info.expect("unsized_info: missing old info for trait upcasting coercion");
if data_a.principal_def_id() == data_b.principal_def_id() {
+ // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables.
return old_info;
}
@@ -162,6 +167,11 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
if let Some(entry_idx) = vptr_entry_idx {
let ptr_ty = cx.type_i8p();
let ptr_align = cx.tcx().data_layout.pointer_align.abi;
+ let vtable_ptr_ty = cx.scalar_pair_element_backend_type(
+ cx.layout_of(cx.tcx().mk_mut_ptr(target)),
+ 1,
+ true,
+ );
let llvtable = bx.pointercast(old_info, bx.type_ptr_to(ptr_ty));
let gep = bx.inbounds_gep(
ptr_ty,
@@ -172,7 +182,7 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
bx.nonnull_metadata(new_vptr);
// VTable loads are invariant.
bx.set_invariant_load(new_vptr);
- new_vptr
+ bx.pointercast(new_vptr, vtable_ptr_ty)
} else {
old_info
}
@@ -388,15 +398,14 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
let main_llfn = cx.get_fn_addr(instance);
- let use_start_lang_item = EntryFnType::Start != entry_type;
- let entry_fn = create_entry_fn::<Bx>(cx, main_llfn, main_def_id, use_start_lang_item);
+ let entry_fn = create_entry_fn::<Bx>(cx, main_llfn, main_def_id, entry_type);
return Some(entry_fn);
fn create_entry_fn<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
cx: &'a Bx::CodegenCx,
rust_main: Bx::Value,
rust_main_def_id: DefId,
- use_start_lang_item: bool,
+ entry_type: EntryFnType,
) -> Bx::Function {
// The entry function is either `int main(void)` or `int main(int argc, char **argv)`,
// depending on whether the target needs `argc` and `argv` to be passed in.
@@ -441,7 +450,7 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
let i8pp_ty = cx.type_ptr_to(cx.type_i8p());
let (arg_argc, arg_argv) = get_argc_argv(cx, &mut bx);
- let (start_fn, start_ty, args) = if use_start_lang_item {
+ let (start_fn, start_ty, args) = if let EntryFnType::Main { sigpipe } = entry_type {
let start_def_id = cx.tcx().require_lang_item(LangItem::Start, None);
let start_fn = cx.get_fn_addr(
ty::Instance::resolve(
@@ -453,8 +462,13 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
.unwrap()
.unwrap(),
);
- let start_ty = cx.type_func(&[cx.val_ty(rust_main), isize_ty, i8pp_ty], isize_ty);
- (start_fn, start_ty, vec![rust_main, arg_argc, arg_argv])
+
+ let i8_ty = cx.type_i8();
+ let arg_sigpipe = bx.const_u8(sigpipe);
+
+ let start_ty =
+ cx.type_func(&[cx.val_ty(rust_main), isize_ty, i8pp_ty, i8_ty], isize_ty);
+ (start_fn, start_ty, vec![rust_main, arg_argc, arg_argv, arg_sigpipe])
} else {
debug!("using user-defined start fn");
let start_ty = cx.type_func(&[isize_ty, i8pp_ty], isize_ty);
@@ -810,21 +824,16 @@ impl CrateInfo {
crate_name: Default::default(),
used_crates,
used_crate_source: Default::default(),
- lang_item_to_crate: Default::default(),
- missing_lang_items: Default::default(),
dependency_formats: tcx.dependency_formats(()).clone(),
windows_subsystem,
natvis_debugger_visualizers: Default::default(),
};
- let lang_items = tcx.lang_items();
-
let crates = tcx.crates(());
let n_crates = crates.len();
info.native_libraries.reserve(n_crates);
info.crate_name.reserve(n_crates);
info.used_crate_source.reserve(n_crates);
- info.missing_lang_items.reserve(n_crates);
for &cnum in crates.iter() {
info.native_libraries
@@ -842,17 +851,41 @@ impl CrateInfo {
if tcx.is_no_builtins(cnum) {
info.is_no_builtins.insert(cnum);
}
- let missing = tcx.missing_lang_items(cnum);
- for &item in missing.iter() {
- if let Ok(id) = lang_items.require(item) {
- info.lang_item_to_crate.insert(item, id.krate);
- }
- }
+ }
- // No need to look for lang items that don't actually need to exist.
- let missing =
- missing.iter().cloned().filter(|&l| lang_items::required(tcx, l)).collect();
- info.missing_lang_items.insert(cnum, missing);
+ // Handle circular dependencies in the standard library.
+ // See comment before `add_linked_symbol_object` function for the details.
+ // If global LTO is enabled then almost everything (*) is glued into a single object file,
+ // so this logic is not necessary and can cause issues on some targets (due to weak lang
+ // item symbols being "privatized" to that object file), so we disable it.
+ // (*) Native libs, and `#[compiler_builtins]` and `#[no_builtins]` crates are not glued,
+ // and we assume that they cannot define weak lang items. This is not currently enforced
+ // by the compiler, but that's ok because all this stuff is unstable anyway.
+ let target = &tcx.sess.target;
+ if !are_upstream_rust_objects_already_included(tcx.sess) {
+ let missing_weak_lang_items: FxHashSet<&Symbol> = info
+ .used_crates
+ .iter()
+ .flat_map(|cnum| {
+ tcx.missing_lang_items(*cnum)
+ .iter()
+ .filter(|l| lang_items::required(tcx, **l))
+ .filter_map(|item| WEAK_ITEMS_SYMBOLS.get(item))
+ })
+ .collect();
+ let prefix = if target.is_like_windows && target.arch == "x86" { "_" } else { "" };
+ info.linked_symbols
+ .iter_mut()
+ .filter(|(crate_type, _)| {
+ !matches!(crate_type, CrateType::Rlib | CrateType::Staticlib)
+ })
+ .for_each(|(_, linked_symbols)| {
+ linked_symbols.extend(
+ missing_weak_lang_items
+ .iter()
+ .map(|item| (format!("{prefix}{item}"), SymbolExportKind::Text)),
+ )
+ });
}
let embed_visualizers = tcx.sess.crate_types().iter().any(|&crate_type| match crate_type {
@@ -873,7 +906,7 @@ impl CrateInfo {
}
});
- if tcx.sess.target.is_like_msvc && embed_visualizers {
+ if target.is_like_msvc && embed_visualizers {
info.natvis_debugger_visualizers =
collect_debugger_visualizers_transitive(tcx, DebuggerVisualizerType::Natvis);
}
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 8cd5a0fc2..135ed680d 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -18,11 +18,10 @@ use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathD
use rustc_hir::{AsyncGeneratorKind, GeneratorKind, Mutability};
use rustc_middle::ty::layout::{IntegerExt, TyAndLayout};
use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
-use rustc_middle::ty::{self, ExistentialProjection, GeneratorSubsts, ParamEnv, Ty, TyCtxt};
-use rustc_target::abi::{Integer, TagEncoding, Variants};
+use rustc_middle::ty::{self, ExistentialProjection, ParamEnv, Ty, TyCtxt};
+use rustc_target::abi::Integer;
use smallvec::SmallVec;
-use std::borrow::Cow;
use std::fmt::Write;
use crate::debuginfo::wants_c_like_enum_debuginfo;
@@ -98,7 +97,6 @@ fn push_debuginfo_type_name<'tcx>(
if let Some(ty_and_layout) = layout_for_cpp_like_fallback {
msvc_enum_fallback(
- tcx,
ty_and_layout,
&|output, visited| {
push_item_name(tcx, def.did(), true, output);
@@ -391,11 +389,10 @@ fn push_debuginfo_type_name<'tcx>(
// Name will be "{closure_env#0}<T1, T2, ...>", "{generator_env#0}<T1, T2, ...>", or
// "{async_fn_env#0}<T1, T2, ...>", etc.
// In the case of cpp-like debuginfo, the name additionally gets wrapped inside of
- // an artificial `enum$<>` type, as defined in msvc_enum_fallback().
+ // an artificial `enum2$<>` type, as defined in msvc_enum_fallback().
if cpp_like_debuginfo && t.is_generator() {
let ty_and_layout = tcx.layout_of(ParamEnv::reveal_all().and(t)).unwrap();
msvc_enum_fallback(
- tcx,
ty_and_layout,
&|output, visited| {
push_closure_or_generator_name(tcx, def_id, substs, true, output, visited);
@@ -428,58 +425,17 @@ fn push_debuginfo_type_name<'tcx>(
/// MSVC names enums differently than other platforms so that the debugging visualization
// format (natvis) is able to understand enums and render the active variant correctly in the
- // debugger. For more information, look in `src/etc/natvis/intrinsic.natvis` and
- // `EnumMemberDescriptionFactor::create_member_descriptions`.
+ // debugger. For more information, look in
+ // rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs.
fn msvc_enum_fallback<'tcx>(
- tcx: TyCtxt<'tcx>,
ty_and_layout: TyAndLayout<'tcx>,
push_inner: &dyn Fn(/*output*/ &mut String, /*visited*/ &mut FxHashSet<Ty<'tcx>>),
output: &mut String,
visited: &mut FxHashSet<Ty<'tcx>>,
) {
debug_assert!(!wants_c_like_enum_debuginfo(ty_and_layout));
- let ty = ty_and_layout.ty;
-
- output.push_str("enum$<");
+ output.push_str("enum2$<");
push_inner(output, visited);
-
- let variant_name = |variant_index| match ty.kind() {
- ty::Adt(adt_def, _) => {
- debug_assert!(adt_def.is_enum());
- Cow::from(adt_def.variant(variant_index).name.as_str())
- }
- ty::Generator(..) => GeneratorSubsts::variant_name(variant_index),
- _ => unreachable!(),
- };
-
- if let Variants::Multiple {
- tag_encoding: TagEncoding::Niche { dataful_variant, .. },
- tag,
- variants,
- ..
- } = &ty_and_layout.variants
- {
- let dataful_variant_layout = &variants[*dataful_variant];
-
- // calculate the range of values for the dataful variant
- let dataful_discriminant_range =
- dataful_variant_layout.largest_niche().unwrap().valid_range;
-
- let min = dataful_discriminant_range.start;
- let min = tag.size(&tcx).truncate(min);
-
- let max = dataful_discriminant_range.end;
- let max = tag.size(&tcx).truncate(max);
-
- let dataful_variant_name = variant_name(*dataful_variant);
- write!(output, ", {}, {}, {}", min, max, dataful_variant_name).unwrap();
- } else if let Variants::Single { index: variant_idx } = &ty_and_layout.variants {
- // Uninhabited enums can't be constructed and should never need to be visualized so
- // skip this step for them.
- if !ty_and_layout.abi.is_uninhabited() {
- write!(output, ", {}", variant_name(*variant_idx)).unwrap();
- }
- }
push_close_angle_bracket(true, output);
}
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 1802eedf1..e736b2aba 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -1,7 +1,7 @@
#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
#![feature(box_patterns)]
#![feature(try_blocks)]
-#![feature(let_else)]
+#![cfg_attr(bootstrap, feature(let_else))]
#![feature(once_cell)]
#![feature(associated_type_bounds)]
#![feature(strict_provenance)]
@@ -25,7 +25,6 @@ use rustc_ast as ast;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_data_structures::sync::Lrc;
use rustc_hir::def_id::CrateNum;
-use rustc_hir::LangItem;
use rustc_middle::dep_graph::WorkProduct;
use rustc_middle::middle::dependency_format::Dependencies;
use rustc_middle::middle::exported_symbols::SymbolExportKind;
@@ -113,6 +112,7 @@ bitflags::bitflags! {
pub struct NativeLib {
pub kind: NativeLibKind,
pub name: Option<Symbol>,
+ pub filename: Option<Symbol>,
pub cfg: Option<ast::MetaItem>,
pub verbatim: Option<bool>,
pub dll_imports: Vec<cstore::DllImport>,
@@ -122,6 +122,7 @@ impl From<&cstore::NativeLib> for NativeLib {
fn from(lib: &cstore::NativeLib) -> Self {
NativeLib {
kind: lib.kind,
+ filename: lib.filename,
name: lib.name,
cfg: lib.cfg.clone(),
verbatim: lib.verbatim,
@@ -152,8 +153,6 @@ pub struct CrateInfo {
pub used_libraries: Vec<NativeLib>,
pub used_crate_source: FxHashMap<CrateNum, Lrc<CrateSource>>,
pub used_crates: Vec<CrateNum>,
- pub lang_item_to_crate: FxHashMap<LangItem, CrateNum>,
- pub missing_lang_items: FxHashMap<CrateNum, Vec<LangItem>>,
pub dependency_formats: Lrc<Dependencies>,
pub windows_subsystem: Option<String>,
pub natvis_debugger_visualizers: BTreeSet<DebuggerVisualizerFile>,
@@ -168,6 +167,13 @@ pub struct CodegenResults {
pub crate_info: CrateInfo,
}
+pub enum CodegenErrors<'a> {
+ WrongFileType,
+ EmptyVersionNumber,
+ EncodingVersionMismatch { version_array: String, rlink_version: u32 },
+ RustcVersionMismatch { rustc_version: String, current_version: &'a str },
+}
+
pub fn provide(providers: &mut Providers) {
crate::back::symbol_export::provide(providers);
crate::base::provide(providers);
@@ -212,30 +218,34 @@ impl CodegenResults {
encoder.finish()
}
- pub fn deserialize_rlink(data: Vec<u8>) -> Result<Self, String> {
+ pub fn deserialize_rlink<'a>(data: Vec<u8>) -> Result<Self, CodegenErrors<'a>> {
// The Decodable machinery is not used here because it panics if the input data is invalid
// and because its internal representation may change.
if !data.starts_with(RLINK_MAGIC) {
- return Err("The input does not look like a .rlink file".to_string());
+ return Err(CodegenErrors::WrongFileType);
}
let data = &data[RLINK_MAGIC.len()..];
if data.len() < 4 {
- return Err("The input does not contain version number".to_string());
+ return Err(CodegenErrors::EmptyVersionNumber);
}
let mut version_array: [u8; 4] = Default::default();
version_array.copy_from_slice(&data[..4]);
if u32::from_be_bytes(version_array) != RLINK_VERSION {
- return Err(".rlink file was produced with encoding version {version_array}, but the current version is {RLINK_VERSION}".to_string());
+ return Err(CodegenErrors::EncodingVersionMismatch {
+ version_array: String::from_utf8_lossy(&version_array).to_string(),
+ rlink_version: RLINK_VERSION,
+ });
}
let mut decoder = MemDecoder::new(&data[4..], 0);
let rustc_version = decoder.read_str();
let current_version = RUSTC_VERSION.unwrap();
if rustc_version != current_version {
- return Err(format!(
- ".rlink file was produced by rustc version {rustc_version}, but the current version is {current_version}."
- ));
+ return Err(CodegenErrors::RustcVersionMismatch {
+ rustc_version: rustc_version.to_string(),
+ current_version,
+ });
}
let codegen_results = CodegenResults::decode(&mut decoder);
diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs
index 27d791d90..cae46ebd2 100644
--- a/compiler/rustc_codegen_ssa/src/meth.rs
+++ b/compiler/rustc_codegen_ssa/src/meth.rs
@@ -1,6 +1,6 @@
use crate::traits::*;
-use rustc_middle::ty::{self, subst::GenericArgKind, ExistentialPredicate, Ty, TyCtxt};
+use rustc_middle::ty::{self, subst::GenericArgKind, Ty};
use rustc_session::config::Lto;
use rustc_symbol_mangling::typeid_for_trait_ref;
use rustc_target::abi::call::FnAbi;
@@ -29,7 +29,7 @@ impl<'a, 'tcx> VirtualIndex {
&& bx.cx().sess().lto() == Lto::Fat
{
let typeid =
- bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), get_trait_ref(bx.tcx(), ty)));
+ bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), expect_dyn_trait_in_self(ty)));
let vtable_byte_offset = self.0 * bx.data_layout().pointer_size.bytes();
let type_checked_load = bx.type_checked_load(llvtable, vtable_byte_offset, typeid);
let func = bx.extract_value(type_checked_load, 0);
@@ -64,17 +64,13 @@ impl<'a, 'tcx> VirtualIndex {
}
}
-fn get_trait_ref<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> {
+/// This takes a valid `self` receiver type and extracts the principal trait
+/// ref of the type.
+fn expect_dyn_trait_in_self<'tcx>(ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> {
for arg in ty.peel_refs().walk() {
if let GenericArgKind::Type(ty) = arg.unpack() {
- if let ty::Dynamic(trait_refs, _) = ty.kind() {
- return trait_refs[0].map_bound(|trait_ref| match trait_ref {
- ExistentialPredicate::Trait(tr) => tr,
- ExistentialPredicate::Projection(proj) => proj.trait_ref(tcx),
- ExistentialPredicate::AutoTrait(_) => {
- bug!("auto traits don't have functions")
- }
- });
+ if let ty::Dynamic(data, _, _) = ty.kind() {
+ return data.principal().expect("expected principal trait object");
}
}
}
@@ -90,6 +86,7 @@ fn get_trait_ref<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::PolyExistentialTr
/// The `trait_ref` encodes the erased self type. Hence if we are
/// making an object `Foo<dyn Trait>` from a value of type `Foo<T>`, then
/// `trait_ref` would map `T: Trait`.
+#[instrument(level = "debug", skip(cx))]
pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
cx: &Cx,
ty: Ty<'tcx>,
@@ -97,8 +94,6 @@ pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
) -> Cx::Value {
let tcx = cx.tcx();
- debug!("get_vtable(ty={:?}, trait_ref={:?})", ty, trait_ref);
-
// Check the cache.
if let Some(&val) = cx.vtables().borrow().get(&(ty, trait_ref)) {
return val;
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index 24da48ead..c7617d2e4 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -266,7 +266,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
result: &mut IndexVec<mir::BasicBlock, CleanupKind>,
mir: &mir::Body<'tcx>,
) {
- for (bb, data) in mir.basic_blocks().iter_enumerated() {
+ for (bb, data) in mir.basic_blocks.iter_enumerated() {
match data.terminator().kind {
TerminatorKind::Goto { .. }
| TerminatorKind::Resume
@@ -296,7 +296,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
}
fn propagate<'tcx>(result: &mut IndexVec<mir::BasicBlock, CleanupKind>, mir: &mir::Body<'tcx>) {
- let mut funclet_succs = IndexVec::from_elem(None, mir.basic_blocks());
+ let mut funclet_succs = IndexVec::from_elem(None, &mir.basic_blocks);
let mut set_successor = |funclet: mir::BasicBlock, succ| match funclet_succs[funclet] {
ref mut s @ None => {
@@ -359,7 +359,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
}
}
- let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, mir.basic_blocks());
+ let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, &mir.basic_blocks);
discover_masters(&mut result, mir);
propagate(&mut result, mir);
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 3eee58d9d..a6b226ef7 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -13,15 +13,14 @@ use rustc_ast as ast;
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_hir::lang_items::LangItem;
use rustc_index::vec::Idx;
-use rustc_middle::mir::AssertKind;
-use rustc_middle::mir::{self, SwitchTargets};
+use rustc_middle::mir::{self, AssertKind, SwitchTargets};
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
use rustc_middle::ty::{self, Instance, Ty, TypeVisitable};
use rustc_span::source_map::Span;
use rustc_span::{sym, Symbol};
use rustc_symbol_mangling::typeid::typeid_for_fnabi;
-use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
+use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode, Reg};
use rustc_target::abi::{self, HasDataLayout, WrappingRange};
use rustc_target::spec::abi::Abi;
@@ -324,7 +323,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
bx.unreachable();
return;
}
- let llval = match self.fn_abi.ret.mode {
+ let llval = match &self.fn_abi.ret.mode {
PassMode::Ignore | PassMode::Indirect { .. } => {
bx.ret_void();
return;
@@ -339,7 +338,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}
- PassMode::Cast(cast_ty) => {
+ PassMode::Cast(cast_ty, _) => {
let op = match self.locals[mir::RETURN_PLACE] {
LocalRef::Operand(Some(op)) => op,
LocalRef::Operand(None) => bug!("use of return before def"),
@@ -360,7 +359,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
llval
}
};
- let ty = bx.cast_backend_type(&cast_ty);
+ let ty = bx.cast_backend_type(cast_ty);
let addr = bx.pointercast(llslot, bx.type_ptr_to(ty));
bx.load(ty, addr, self.fn_abi.ret.layout.align.abi)
}
@@ -368,6 +367,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
bx.ret(llval);
}
+ #[tracing::instrument(level = "trace", skip(self, helper, bx))]
fn codegen_drop_terminator(
&mut self,
helper: TerminatorCodegenHelper<'tcx>,
@@ -398,13 +398,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let (drop_fn, fn_abi) = match ty.kind() {
// FIXME(eddyb) perhaps move some of this logic into
// `Instance::resolve_drop_in_place`?
- ty::Dynamic(..) => {
+ ty::Dynamic(_, _, ty::Dyn) => {
+ // IN THIS ARM, WE HAVE:
+ // ty = *mut (dyn Trait)
+ // which is: exists<T> ( *mut T, Vtable<T: Trait> )
+ // args[0] args[1]
+ //
+ // args = ( Data, Vtable )
+ // |
+ // v
+ // /-------\
+ // | ... |
+ // \-------/
+ //
let virtual_drop = Instance {
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
substs: drop_fn.substs,
};
+ debug!("ty = {:?}", ty);
+ debug!("drop_fn = {:?}", drop_fn);
+ debug!("args = {:?}", args);
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
let vtable = args[1];
+ // Truncate vtable off of args list
args = &args[..1];
(
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
@@ -412,6 +428,51 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
fn_abi,
)
}
+ ty::Dynamic(_, _, ty::DynStar) => {
+ // IN THIS ARM, WE HAVE:
+ // ty = *mut (dyn* Trait)
+ // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
+ //
+ // args = [ * ]
+ // |
+ // v
+ // ( Data, Vtable )
+ // |
+ // v
+ // /-------\
+ // | ... |
+ // \-------/
+ //
+ //
+ // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
+ //
+ // data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer)
+ // vtable = (*args[0]).1 // loads the vtable out
+ // (data, vtable) // an equivalent Rust `*mut dyn Trait`
+ //
+ // SO THEN WE CAN USE THE ABOVE CODE.
+ let virtual_drop = Instance {
+ def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+ substs: drop_fn.substs,
+ };
+ debug!("ty = {:?}", ty);
+ debug!("drop_fn = {:?}", drop_fn);
+ debug!("args = {:?}", args);
+ let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
+ let data = args[0];
+ let data_ty = bx.cx().backend_type(place.layout);
+ let vtable_ptr =
+ bx.gep(data_ty, data, &[bx.cx().const_i32(0), bx.cx().const_i32(1)]);
+ let vtable = bx.load(bx.type_i8p(), vtable_ptr, abi::Align::ONE);
+ // Truncate vtable off of args list
+ args = &args[..1];
+ debug!("args' = {:?}", args);
+ (
+ meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
+ .get_fn(&mut bx, vtable, ty, &fn_abi),
+ fn_abi,
+ )
+ }
_ => (bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty())),
};
helper.do_call(
@@ -798,58 +859,78 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let mut op = self.codegen_operand(&mut bx, arg);
if let (0, Some(ty::InstanceDef::Virtual(_, idx))) = (i, def) {
- if let Pair(..) = op.val {
- // In the case of Rc<Self>, we need to explicitly pass a
- // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
- // that is understood elsewhere in the compiler as a method on
- // `dyn Trait`.
- // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
- // we get a value of a built-in pointer type
- 'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
- && !op.layout.ty.is_region_ptr()
- {
- for i in 0..op.layout.fields.count() {
- let field = op.extract_field(&mut bx, i);
- if !field.layout.is_zst() {
- // we found the one non-zero-sized field that is allowed
- // now find *its* non-zero-sized field, or stop if it's a
- // pointer
- op = field;
- continue 'descend_newtypes;
+ match op.val {
+ Pair(data_ptr, meta) => {
+ // In the case of Rc<Self>, we need to explicitly pass a
+ // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
+ // that is understood elsewhere in the compiler as a method on
+ // `dyn Trait`.
+ // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
+ // we get a value of a built-in pointer type
+ 'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
+ && !op.layout.ty.is_region_ptr()
+ {
+ for i in 0..op.layout.fields.count() {
+ let field = op.extract_field(&mut bx, i);
+ if !field.layout.is_zst() {
+ // we found the one non-zero-sized field that is allowed
+ // now find *its* non-zero-sized field, or stop if it's a
+ // pointer
+ op = field;
+ continue 'descend_newtypes;
+ }
}
+
+ span_bug!(span, "receiver has no non-zero-sized fields {:?}", op);
}
- span_bug!(span, "receiver has no non-zero-sized fields {:?}", op);
+ // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
+ // data pointer and vtable. Look up the method in the vtable, and pass
+ // the data pointer as the first argument
+ llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+ &mut bx,
+ meta,
+ op.layout.ty,
+ &fn_abi,
+ ));
+ llargs.push(data_ptr);
+ continue 'make_args;
}
-
- // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
- // data pointer and vtable. Look up the method in the vtable, and pass
- // the data pointer as the first argument
- match op.val {
- Pair(data_ptr, meta) => {
- llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
- &mut bx,
- meta,
- op.layout.ty,
- &fn_abi,
- ));
- llargs.push(data_ptr);
- continue 'make_args;
+ Ref(data_ptr, Some(meta), _) => {
+ // by-value dynamic dispatch
+ llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+ &mut bx,
+ meta,
+ op.layout.ty,
+ &fn_abi,
+ ));
+ llargs.push(data_ptr);
+ continue;
+ }
+ Immediate(_) => {
+ let ty::Ref(_, ty, _) = op.layout.ty.kind() else {
+ span_bug!(span, "can't codegen a virtual call on {:#?}", op);
+ };
+ if !ty.is_dyn_star() {
+ span_bug!(span, "can't codegen a virtual call on {:#?}", op);
}
- other => bug!("expected a Pair, got {:?}", other),
+ // FIXME(dyn-star): Make sure this is done on a &dyn* receiver
+ let place = op.deref(bx.cx());
+ let data_ptr = place.project_field(&mut bx, 0);
+ let meta_ptr = place.project_field(&mut bx, 1);
+ let meta = bx.load_operand(meta_ptr);
+ llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+ &mut bx,
+ meta.immediate(),
+ op.layout.ty,
+ &fn_abi,
+ ));
+ llargs.push(data_ptr.llval);
+ continue;
+ }
+ _ => {
+ span_bug!(span, "can't codegen a virtual call on {:#?}", op);
}
- } else if let Ref(data_ptr, Some(meta), _) = op.val {
- // by-value dynamic dispatch
- llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
- &mut bx,
- meta,
- op.layout.ty,
- &fn_abi,
- ));
- llargs.push(data_ptr);
- continue;
- } else {
- span_bug!(span, "can't codegen a virtual call on {:?}", op);
}
}
@@ -1161,39 +1242,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
llargs: &mut Vec<Bx::Value>,
arg: &ArgAbi<'tcx, Ty<'tcx>>,
) {
- // Fill padding with undef value, where applicable.
- if let Some(ty) = arg.pad {
- llargs.push(bx.const_undef(bx.reg_backend_type(&ty)))
- }
-
- if arg.is_ignore() {
- return;
- }
-
- if let PassMode::Pair(..) = arg.mode {
- match op.val {
+ match arg.mode {
+ PassMode::Ignore => return,
+ PassMode::Cast(_, true) => {
+ // Fill padding with undef value, where applicable.
+ llargs.push(bx.const_undef(bx.reg_backend_type(&Reg::i32())));
+ }
+ PassMode::Pair(..) => match op.val {
Pair(a, b) => {
llargs.push(a);
llargs.push(b);
return;
}
_ => bug!("codegen_argument: {:?} invalid for pair argument", op),
- }
- } else if arg.is_unsized_indirect() {
- match op.val {
+ },
+ PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => match op.val {
Ref(a, Some(b), _) => {
llargs.push(a);
llargs.push(b);
return;
}
_ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
- }
+ },
+ _ => {}
}
// Force by-ref if we have to load through a cast pointer.
let (mut llval, align, by_ref) = match op.val {
Immediate(_) | Pair(..) => match arg.mode {
- PassMode::Indirect { .. } | PassMode::Cast(_) => {
+ PassMode::Indirect { .. } | PassMode::Cast(..) => {
let scratch = PlaceRef::alloca(bx, arg.layout);
op.val.store(bx, scratch);
(scratch.llval, scratch.align, true)
@@ -1225,8 +1302,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
if by_ref && !arg.is_indirect() {
// Have to load the argument, maybe while casting it.
- if let PassMode::Cast(ty) = arg.mode {
- let llty = bx.cast_backend_type(&ty);
+ if let PassMode::Cast(ty, _) = &arg.mode {
+ let llty = bx.cast_backend_type(ty);
let addr = bx.pointercast(llval, bx.type_ptr_to(llty));
llval = bx.load(llty, addr, align.min(arg.layout.align.abi));
} else {
@@ -1625,7 +1702,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
DirectOperand(index) => {
// If there is a cast, we have to store and reload.
- let op = if let PassMode::Cast(_) = ret_abi.mode {
+ let op = if let PassMode::Cast(..) = ret_abi.mode {
let tmp = PlaceRef::alloca(bx, ret_abi.layout);
tmp.storage_live(bx);
bx.store_arg(&ret_abi, llval, tmp);
diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs
index 9a995fbf6..4c6ab457c 100644
--- a/compiler/rustc_codegen_ssa/src/mir/constant.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs
@@ -25,26 +25,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
constant: &mir::Constant<'tcx>,
) -> Result<ConstValue<'tcx>, ErrorHandled> {
let ct = self.monomorphize(constant.literal);
- let ct = match ct {
- mir::ConstantKind::Ty(ct) => ct,
+ let uv = match ct {
+ mir::ConstantKind::Ty(ct) => match ct.kind() {
+ ty::ConstKind::Unevaluated(uv) => uv.expand(),
+ ty::ConstKind::Value(val) => {
+ return Ok(self.cx.tcx().valtree_to_const_val((ct.ty(), val)));
+ }
+ err => span_bug!(
+ constant.span,
+ "encountered bad ConstKind after monomorphizing: {:?}",
+ err
+ ),
+ },
+ mir::ConstantKind::Unevaluated(uv, _) => uv,
mir::ConstantKind::Val(val, _) => return Ok(val),
};
- match ct.kind() {
- ty::ConstKind::Unevaluated(ct) => self
- .cx
- .tcx()
- .const_eval_resolve(ty::ParamEnv::reveal_all(), ct, None)
- .map_err(|err| {
- self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered");
- err
- }),
- ty::ConstKind::Value(val) => Ok(self.cx.tcx().valtree_to_const_val((ct.ty(), val))),
- err => span_bug!(
- constant.span,
- "encountered bad ConstKind after monomorphizing: {:?}",
- err
- ),
- }
+
+ self.cx.tcx().const_eval_resolve(ty::ParamEnv::reveal_all(), uv, None).map_err(|err| {
+ self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered");
+ err
+ })
}
/// process constant containing SIMD shuffle indices
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
index 8c3186efc..157c1c823 100644
--- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -3,7 +3,7 @@ use rustc_index::vec::IndexVec;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir;
use rustc_middle::ty;
-use rustc_middle::ty::layout::LayoutOf;
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
use rustc_session::config::DebugInfo;
use rustc_span::symbol::{kw, Symbol};
use rustc_span::{BytePos, Span};
@@ -93,15 +93,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
/// In order to have a good line stepping behavior in debugger, we overwrite debug
- /// locations of macro expansions with that of the outermost expansion site
- /// (unless the crate is being compiled with `-Z debug-macros`).
+ /// locations of macro expansions with that of the outermost expansion site (when the macro is
+ /// annotated with `#[collapse_debuginfo]` or when `-Zdebug-macros` is provided).
fn adjust_span_for_debugging(&self, mut span: Span) -> Span {
// Bail out if debug info emission is not enabled.
if self.debug_context.is_none() {
return span;
}
- if span.from_expansion() && !self.cx.sess().opts.unstable_opts.debug_macros {
+ if self.cx.tcx().should_collapse_debuginfo(span) {
// Walk up the macro expansion chain until we reach a non-expanded span.
// We also stop at the function body level because no line stepping can occur
// at the level above that.
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 94ac71a4d..215edbe02 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -77,10 +77,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
let llval = match name {
- sym::assume => {
- bx.assume(args[0].immediate());
- return;
- }
sym::abort => {
bx.abort();
return;
@@ -555,14 +551,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
return;
}
- sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => {
+ sym::ptr_guaranteed_cmp => {
let a = args[0].immediate();
let b = args[1].immediate();
- if name == sym::ptr_guaranteed_eq {
- bx.icmp(IntPredicate::IntEQ, a, b)
- } else {
- bx.icmp(IntPredicate::IntNE, a, b)
- }
+ bx.icmp(IntPredicate::IntEQ, a, b)
}
sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
@@ -597,8 +589,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
};
if !fn_abi.ret.is_ignore() {
- if let PassMode::Cast(ty) = fn_abi.ret.mode {
- let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(&ty));
+ if let PassMode::Cast(ty, _) = &fn_abi.ret.mode {
+ let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(ty));
let ptr = bx.pointercast(result.llval, ptr_llty);
bx.store(llval, ptr, result.align);
} else {
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index 8ee375fa9..2b931bfc9 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -150,13 +150,13 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
let start_llbb = Bx::append_block(cx, llfn, "start");
let mut bx = Bx::build(cx, start_llbb);
- if mir.basic_blocks().iter().any(|bb| bb.is_cleanup) {
+ if mir.basic_blocks.iter().any(|bb| bb.is_cleanup) {
bx.set_personality_fn(cx.eh_personality());
}
let cleanup_kinds = analyze::cleanup_kinds(&mir);
let cached_llbbs: IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>> = mir
- .basic_blocks()
+ .basic_blocks
.indices()
.map(|bb| if bb == mir::START_BLOCK { Some(start_llbb) } else { None })
.collect();
@@ -172,8 +172,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
unreachable_block: None,
double_unwind_guard: None,
cleanup_kinds,
- landing_pads: IndexVec::from_elem(None, mir.basic_blocks()),
- funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks().len()),
+ landing_pads: IndexVec::from_elem(None, &mir.basic_blocks),
+ funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()),
locals: IndexVec::new(),
debug_context,
per_local_var_debug_info: None,
@@ -191,7 +191,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
// errored or at least linted
ErrorHandled::Reported(_) | ErrorHandled::Linted => {}
ErrorHandled::TooGeneric => {
- span_bug!(const_.span, "codgen encountered polymorphic constant: {:?}", err)
+ span_bug!(const_.span, "codegen encountered polymorphic constant: {:?}", err)
}
}
}
@@ -283,7 +283,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
for i in 0..tupled_arg_tys.len() {
let arg = &fx.fn_abi.args[idx];
idx += 1;
- if arg.pad.is_some() {
+ if let PassMode::Cast(_, true) = arg.mode {
llarg_idx += 1;
}
let pr_field = place.project_field(bx, i);
@@ -309,7 +309,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
let arg = &fx.fn_abi.args[idx];
idx += 1;
- if arg.pad.is_some() {
+ if let PassMode::Cast(_, true) = arg.mode {
llarg_idx += 1;
}
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index c612634fc..37b1e0362 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -72,10 +72,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
) -> Self {
let layout = bx.layout_of(ty);
- if layout.is_zst() {
- return OperandRef::new_zst(bx, layout);
- }
-
let val = match val {
ConstValue::Scalar(x) => {
let Abi::Scalar(scalar) = layout.abi else {
@@ -84,10 +80,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
OperandValue::Immediate(llval)
}
- ConstValue::ZeroSized => {
- let llval = bx.zst_to_backend(bx.immediate_backend_type(layout));
- OperandValue::Immediate(llval)
- }
+ ConstValue::ZeroSized => return OperandRef::new_zst(bx, layout),
ConstValue::Slice { data, start, end } => {
let Abi::ScalarPair(a_scalar, _) = layout.abi else {
bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index 268c4d765..13d8f6edd 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -4,7 +4,6 @@ use super::{FunctionCx, LocalRef};
use crate::common::IntPredicate;
use crate::glue;
use crate::traits::*;
-use crate::MemFlags;
use rustc_middle::mir;
use rustc_middle::mir::tcx::PlaceTy;
@@ -245,7 +244,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
};
bx.intcast(tag.immediate(), cast_to, signed)
}
- TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
+ TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
// Rebase from niche values to discriminants, and check
// whether the result is in range for the niche variants.
let niche_llty = bx.cx().immediate_backend_type(tag.layout);
@@ -303,7 +302,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
bx.select(
is_niche,
niche_discr,
- bx.cx().const_uint(cast_to, dataful_variant.as_u32() as u64),
+ bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64),
)
}
}
@@ -338,21 +337,11 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
}
Variants::Multiple {
tag_encoding:
- TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
+ TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
tag_field,
..
} => {
- if variant_index != dataful_variant {
- if bx.cx().sess().target.arch == "arm"
- || bx.cx().sess().target.arch == "aarch64"
- {
- // FIXME(#34427): as workaround for LLVM bug on ARM,
- // use memset of 0 before assigning niche value.
- let fill_byte = bx.cx().const_u8(0);
- let size = bx.cx().const_usize(self.layout.size.bytes());
- bx.memset(self.llval, fill_byte, size, self.align, MemFlags::empty());
- }
-
+ if variant_index != untagged_variant {
let niche = self.project_field(bx, tag_field);
let niche_llty = bx.cx().immediate_backend_type(niche.layout);
let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 26b9fbf44..56852b0fc 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -4,6 +4,7 @@ use super::{FunctionCx, LocalRef};
use crate::base;
use crate::common::{self, IntPredicate};
+use crate::meth::get_vtable;
use crate::traits::*;
use crate::MemFlags;
@@ -87,7 +88,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let size = bx.const_usize(dest.layout.size.bytes());
// Use llvm.memset.p0i8.* to initialize all zero arrays
- if bx.cx().const_to_opt_uint(v) == Some(0) {
+ if bx.cx().const_to_opt_u128(v, false) == Some(0) {
let fill = bx.cx().const_u8(0);
bx.memset(start, fill, size, dest.align, MemFlags::empty());
return bx;
@@ -271,6 +272,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
bug!("unexpected non-pair operand");
}
}
+ mir::CastKind::DynStar => {
+ let data = match operand.val {
+ OperandValue::Ref(_, _, _) => todo!(),
+ OperandValue::Immediate(v) => v,
+ OperandValue::Pair(_, _) => todo!(),
+ };
+ let trait_ref =
+ if let ty::Dynamic(data, _, ty::DynStar) = cast.ty.kind() {
+ data.principal()
+ } else {
+ bug!("Only valid to do a DynStar cast into a DynStar type")
+ };
+ let vtable = get_vtable(bx.cx(), source.ty(self.mir, bx.tcx()), trait_ref);
+ OperandValue::Pair(data, vtable)
+ }
mir::CastKind::Pointer(
PointerCast::MutToConstPointer | PointerCast::ArrayToPointer,
)
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
index f452f2988..1db0fb3a6 100644
--- a/compiler/rustc_codegen_ssa/src/mir/statement.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -1,4 +1,5 @@
use rustc_middle::mir;
+use rustc_middle::mir::NonDivergingIntrinsic;
use super::FunctionCx;
use super::LocalRef;
@@ -73,11 +74,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope);
bx
}
- mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
- ref src,
- ref dst,
- ref count,
- }) => {
+ mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
+ let op_val = self.codegen_operand(&mut bx, op);
+ bx.assume(op_val.immediate());
+ bx
+ }
+ mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
+ mir::CopyNonOverlapping { ref count, ref src, ref dst },
+ )) => {
let dst_val = self.codegen_operand(&mut bx, dst);
let src_val = self.codegen_operand(&mut bx, src);
let count = self.codegen_operand(&mut bx, count).immediate();
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index ecad05185..0e259bcd1 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -210,8 +210,11 @@ const POWERPC_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
("vsx", Some(sym::powerpc_target_feature)),
];
-const MIPS_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] =
- &[("fp64", Some(sym::mips_target_feature)), ("msa", Some(sym::mips_target_feature))];
+const MIPS_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
+ ("fp64", Some(sym::mips_target_feature)),
+ ("msa", Some(sym::mips_target_feature)),
+ ("virt", Some(sym::mips_target_feature)),
+];
const RISCV_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
("m", Some(sym::riscv_target_feature)),
@@ -227,6 +230,10 @@ const RISCV_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
("zhinxmin", Some(sym::riscv_target_feature)),
("zfh", Some(sym::riscv_target_feature)),
("zfhmin", Some(sym::riscv_target_feature)),
+ ("zba", Some(sym::riscv_target_feature)),
+ ("zbb", Some(sym::riscv_target_feature)),
+ ("zbc", Some(sym::riscv_target_feature)),
+ ("zbs", Some(sym::riscv_target_feature)),
("zbkb", Some(sym::riscv_target_feature)),
("zbkc", Some(sym::riscv_target_feature)),
("zbkx", Some(sym::riscv_target_feature)),
diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs
index 9f49749bb..10cf8948b 100644
--- a/compiler/rustc_codegen_ssa/src/traits/builder.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs
@@ -1,6 +1,5 @@
use super::abi::AbiBuilderMethods;
use super::asm::AsmBuilderMethods;
-use super::consts::ConstMethods;
use super::coverageinfo::CoverageInfoBuilderMethods;
use super::debuginfo::DebugInfoBuilderMethods;
use super::intrinsic::IntrinsicCallMethods;
@@ -15,7 +14,6 @@ use crate::mir::operand::OperandRef;
use crate::mir::place::PlaceRef;
use crate::MemFlags;
-use rustc_apfloat::{ieee, Float, Round, Status};
use rustc_middle::ty::layout::{HasParamEnv, TyAndLayout};
use rustc_middle::ty::Ty;
use rustc_span::Span;
@@ -188,8 +186,8 @@ pub trait BuilderMethods<'a, 'tcx>:
fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
- fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
- fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
+ fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+ fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
@@ -223,156 +221,7 @@ pub trait BuilderMethods<'a, 'tcx>:
return if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) };
}
- let try_sat_result =
- if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) };
- if let Some(try_sat_result) = try_sat_result {
- return try_sat_result;
- }
-
- let int_width = self.cx().int_width(int_ty);
- let float_width = self.cx().float_width(float_ty);
- // LLVM's fpto[su]i returns undef when the input x is infinite, NaN, or does not fit into the
- // destination integer type after rounding towards zero. This `undef` value can cause UB in
- // safe code (see issue #10184), so we implement a saturating conversion on top of it:
- // Semantically, the mathematical value of the input is rounded towards zero to the next
- // mathematical integer, and then the result is clamped into the range of the destination
- // integer type. Positive and negative infinity are mapped to the maximum and minimum value of
- // the destination integer type. NaN is mapped to 0.
- //
- // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to
- // a value representable in int_ty.
- // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits.
- // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two.
- // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly
- // representable. Note that this only works if float_ty's exponent range is sufficiently large.
- // f16 or 256 bit integers would break this property. Right now the smallest float type is f32
- // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127.
- // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because
- // we're rounding towards zero, we just get float_ty::MAX (which is always an integer).
- // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX.
- let int_max = |signed: bool, int_width: u64| -> u128 {
- let shift_amount = 128 - int_width;
- if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount }
- };
- let int_min = |signed: bool, int_width: u64| -> i128 {
- if signed { i128::MIN >> (128 - int_width) } else { 0 }
- };
-
- let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) {
- let rounded_min =
- ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero);
- assert_eq!(rounded_min.status, Status::OK);
- let rounded_max =
- ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero);
- assert!(rounded_max.value.is_finite());
- (rounded_min.value.to_bits(), rounded_max.value.to_bits())
- };
- let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) {
- let rounded_min =
- ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero);
- assert_eq!(rounded_min.status, Status::OK);
- let rounded_max =
- ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero);
- assert!(rounded_max.value.is_finite());
- (rounded_min.value.to_bits(), rounded_max.value.to_bits())
- };
- // To implement saturation, we perform the following steps:
- //
- // 1. Cast x to an integer with fpto[su]i. This may result in undef.
- // 2. Compare x to f_min and f_max, and use the comparison results to select:
- // a) int_ty::MIN if x < f_min or x is NaN
- // b) int_ty::MAX if x > f_max
- // c) the result of fpto[su]i otherwise
- // 3. If x is NaN, return 0.0, otherwise return the result of step 2.
- //
- // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the
- // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of
- // undef does not introduce any non-determinism either.
- // More importantly, the above procedure correctly implements saturating conversion.
- // Proof (sketch):
- // If x is NaN, 0 is returned by definition.
- // Otherwise, x is finite or infinite and thus can be compared with f_min and f_max.
- // This yields three cases to consider:
- // (1) if x in [f_min, f_max], the result of fpto[su]i is returned, which agrees with
- // saturating conversion for inputs in that range.
- // (2) if x > f_max, then x is larger than int_ty::MAX. This holds even if f_max is rounded
- // (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger
- // than int_ty::MAX. Because x is larger than int_ty::MAX, the return value of int_ty::MAX
- // is correct.
- // (3) if x < f_min, then x is smaller than int_ty::MIN. As shown earlier, f_min exactly equals
- // int_ty::MIN and therefore the return value of int_ty::MIN is correct.
- // QED.
-
- let float_bits_to_llval = |bx: &mut Self, bits| {
- let bits_llval = match float_width {
- 32 => bx.cx().const_u32(bits as u32),
- 64 => bx.cx().const_u64(bits as u64),
- n => bug!("unsupported float width {}", n),
- };
- bx.bitcast(bits_llval, float_ty)
- };
- let (f_min, f_max) = match float_width {
- 32 => compute_clamp_bounds_single(signed, int_width),
- 64 => compute_clamp_bounds_double(signed, int_width),
- n => bug!("unsupported float width {}", n),
- };
- let f_min = float_bits_to_llval(self, f_min);
- let f_max = float_bits_to_llval(self, f_max);
- let int_max = self.cx().const_uint_big(int_ty, int_max(signed, int_width));
- let int_min = self.cx().const_uint_big(int_ty, int_min(signed, int_width) as u128);
- let zero = self.cx().const_uint(int_ty, 0);
-
- // If we're working with vectors, constants must be "splatted": the constant is duplicated
- // into each lane of the vector. The algorithm stays the same, we are just using the
- // same constant across all lanes.
- let maybe_splat = |bx: &mut Self, val| {
- if bx.cx().type_kind(dest_ty) == TypeKind::Vector {
- bx.vector_splat(bx.vector_length(dest_ty), val)
- } else {
- val
- }
- };
- let f_min = maybe_splat(self, f_min);
- let f_max = maybe_splat(self, f_max);
- let int_max = maybe_splat(self, int_max);
- let int_min = maybe_splat(self, int_min);
- let zero = maybe_splat(self, zero);
-
- // Step 1 ...
- let fptosui_result = if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) };
- let less_or_nan = self.fcmp(RealPredicate::RealULT, x, f_min);
- let greater = self.fcmp(RealPredicate::RealOGT, x, f_max);
-
- // Step 2: We use two comparisons and two selects, with %s1 being the
- // result:
- // %less_or_nan = fcmp ult %x, %f_min
- // %greater = fcmp olt %x, %f_max
- // %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result
- // %s1 = select %greater, int_ty::MAX, %s0
- // Note that %less_or_nan uses an *unordered* comparison. This
- // comparison is true if the operands are not comparable (i.e., if x is
- // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if
- // x is NaN.
- //
- // Performance note: Unordered comparison can be lowered to a "flipped"
- // comparison and a negation, and the negation can be merged into the
- // select. Therefore, it not necessarily any more expensive than an
- // ordered ("normal") comparison. Whether these optimizations will be
- // performed is ultimately up to the backend, but at least x86 does
- // perform them.
- let s0 = self.select(less_or_nan, int_min, fptosui_result);
- let s1 = self.select(greater, int_max, s0);
-
- // Step 3: NaN replacement.
- // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN.
- // Therefore we only need to execute this step for signed integer types.
- if signed {
- // LLVM has no isNaN predicate, so we use (x == x) instead
- let cmp = self.fcmp(RealPredicate::RealOEQ, x, x);
- self.select(cmp, s1, zero)
- } else {
- s1
- }
+ if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) }
}
fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs
index 8a91d4735..fdc7a30e8 100644
--- a/compiler/rustc_codegen_ssa/src/traits/consts.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs
@@ -29,7 +29,6 @@ pub trait ConstMethods<'tcx>: BackendTypes {
fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value;
fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value;
- fn zst_to_backend(&self, llty: Self::Type) -> Self::Value;
fn from_const_alloc(
&self,
layout: TyAndLayout<'tcx>,