Adding debian version 1.65.0+dfsg1-2.debian/1.65.0+dfsg1-2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

23 files changed, 646 insertions, 681 deletions

diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs
index 0d2aa483d..bb76ca5d2 100644
--- a/compiler/rustc_codegen_ssa/src/back/archive.rs
+++ b/compiler/rustc_codegen_ssa/src/back/archive.rs
@@ -1,44 +1,16 @@
+use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::memmap::Mmap;
 use rustc_session::cstore::DllImport;
 use rustc_session::Session;
+use rustc_span::symbol::Symbol;
 
+use object::read::archive::ArchiveFile;
+
+use std::fmt::Display;
+use std::fs::File;
 use std::io;
 use std::path::{Path, PathBuf};
 
-pub(super) fn find_library(
-    name: &str,
-    verbatim: bool,
-    search_paths: &[PathBuf],
-    sess: &Session,
-) -> PathBuf {
-    // On Windows, static libraries sometimes show up as libfoo.a and other
-    // times show up as foo.lib
-    let oslibname = if verbatim {
-        name.to_string()
-    } else {
-        format!("{}{}{}", sess.target.staticlib_prefix, name, sess.target.staticlib_suffix)
-    };
-    let unixlibname = format!("lib{}.a", name);
-
-    for path in search_paths {
-        debug!("looking for {} inside {:?}", name, path);
-        let test = path.join(&oslibname);
-        if test.exists() {
-            return test;
-        }
-        if oslibname != unixlibname {
-            let test = path.join(&unixlibname);
-            if test.exists() {
-                return test;
-            }
-        }
-    }
-    sess.fatal(&format!(
-        "could not find native static library `{}`, \
-                         perhaps an -L flag is missing?",
-        name
-    ));
-}
-
 pub trait ArchiveBuilderBuilder {
     fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a>;
 
@@ -53,7 +25,38 @@ pub trait ArchiveBuilderBuilder {
         lib_name: &str,
         dll_imports: &[DllImport],
         tmpdir: &Path,
+        is_direct_dependency: bool,
     ) -> PathBuf;
+
+    fn extract_bundled_libs(
+        &self,
+        rlib: &Path,
+        outdir: &Path,
+        bundled_lib_file_names: &FxHashSet<Symbol>,
+    ) -> Result<(), String> {
+        let message = |msg: &str, e: &dyn Display| format!("{} '{}': {}", msg, &rlib.display(), e);
+        let archive_map = unsafe {
+            Mmap::map(File::open(rlib).map_err(|e| message("failed to open file", &e))?)
+                .map_err(|e| message("failed to mmap file", &e))?
+        };
+        let archive = ArchiveFile::parse(&*archive_map)
+            .map_err(|e| message("failed to parse archive", &e))?;
+
+        for entry in archive.members() {
+            let entry = entry.map_err(|e| message("failed to read entry", &e))?;
+            let data = entry
+                .data(&*archive_map)
+                .map_err(|e| message("failed to get data from archive member", &e))?;
+            let name = std::str::from_utf8(entry.name())
+                .map_err(|e| message("failed to convert name", &e))?;
+            if !bundled_lib_file_names.contains(&Symbol::intern(name)) {
+                continue; // We need to extract only native libraries.
+            }
+            std::fs::write(&outdir.join(&name), data)
+                .map_err(|e| message("failed to write file", &e))?;
+        }
+        Ok(())
+    }
 }
 
 pub trait ArchiveBuilder<'a> {
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 63207803e..6cce95427 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -1,11 +1,13 @@
 use rustc_arena::TypedArena;
 use rustc_ast::CRATE_NODE_ID;
-use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
+use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_errors::{ErrorGuaranteed, Handler};
 use rustc_fs_util::fix_windows_verbatim_for_gcc;
 use rustc_hir::def_id::CrateNum;
+use rustc_metadata::find_native_static_library;
 use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME};
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
@@ -20,11 +22,11 @@ use rustc_session::utils::NativeLibKind;
 use rustc_session::{filesearch, Session};
 use rustc_span::symbol::Symbol;
 use rustc_span::DebuggerVisualizerFile;
-use rustc_target::spec::crt_objects::{CrtObjects, CrtObjectsFallback};
+use rustc_target::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault};
 use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo};
 use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Target};
 
-use super::archive::{find_library, ArchiveBuilder, ArchiveBuilderBuilder};
+use super::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
 use super::command::Command;
 use super::linker::{self, Linker};
 use super::metadata::{create_rmeta_file, MetadataPosition};
@@ -44,7 +46,7 @@ use std::io::{BufWriter, Write};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::process::{ExitStatus, Output, Stdio};
-use std::{ascii, char, env, fmt, fs, io, mem, str};
+use std::{env, fmt, fs, io, mem, str};
 
 pub fn ensure_removed(diag_handler: &Handler, path: &Path) {
     if let Err(e) = fs::remove_file(path) {
@@ -307,6 +309,9 @@ fn link_rlib<'a>(
         }
     }
 
+    // Used if packed_bundled_libs flag enabled.
+    let mut packed_bundled_libs = Vec::new();
+
     // Note that in this loop we are ignoring the value of `lib.cfg`. That is,
     // we may not be configured to actually include a static library if we're
     // adding it here. That's because later when we consume this rlib we'll
@@ -326,6 +331,8 @@ fn link_rlib<'a>(
     for lib in codegen_results.crate_info.used_libraries.iter() {
         match lib.kind {
             NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
+                if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.packed_bundled_libs => {}
+            NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
                 if flavor == RlibFlavor::Normal =>
             {
                 // Don't allow mixing +bundle with +whole_archive since an rlib may contain
@@ -348,7 +355,16 @@ fn link_rlib<'a>(
         }
         if let Some(name) = lib.name {
             let location =
-                find_library(name.as_str(), lib.verbatim.unwrap_or(false), &lib_search_paths, sess);
+                find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
+            if sess.opts.unstable_opts.packed_bundled_libs && flavor == RlibFlavor::Normal {
+                packed_bundled_libs.push(find_native_static_library(
+                    lib.filename.unwrap().as_str(),
+                    Some(true),
+                    &lib_search_paths,
+                    sess,
+                ));
+                continue;
+            }
             ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|e| {
                 sess.fatal(&format!(
                     "failed to add native library {}: {}",
@@ -360,13 +376,14 @@ fn link_rlib<'a>(
     }
 
     for (raw_dylib_name, raw_dylib_imports) in
-        collate_raw_dylibs(sess, &codegen_results.crate_info.used_libraries)?
+        collate_raw_dylibs(sess, codegen_results.crate_info.used_libraries.iter())?
     {
         let output_path = archive_builder_builder.create_dll_import_lib(
             sess,
             &raw_dylib_name,
             &raw_dylib_imports,
             tmpdir.as_ref(),
+            true,
         );
 
         ab.add_archive(&output_path, Box::new(|_| false)).unwrap_or_else(|e| {
@@ -403,6 +420,12 @@ fn link_rlib<'a>(
         ab.add_file(&trailing_metadata);
     }
 
+    // Add all bundled static native library dependencies.
+    // Archives added to the end of .rlib archive, see comment above for the reason.
+    for lib in packed_bundled_libs {
+        ab.add_file(&lib)
+    }
+
     return Ok(ab);
 }
 
@@ -412,9 +435,9 @@ fn link_rlib<'a>(
 /// then the CodegenResults value contains one NativeLib instance for each block.  However, the
 /// linker appears to expect only a single import library for each library used, so we need to
 /// collate the symbols together by library name before generating the import libraries.
-fn collate_raw_dylibs(
-    sess: &Session,
-    used_libraries: &[NativeLib],
+fn collate_raw_dylibs<'a, 'b>(
+    sess: &'a Session,
+    used_libraries: impl IntoIterator<Item = &'b NativeLib>,
 ) -> Result<Vec<(String, Vec<DllImport>)>, ErrorGuaranteed> {
     // Use index maps to preserve original order of imports and libraries.
     let mut dylib_table = FxIndexMap::<String, FxIndexMap<Symbol, &DllImport>>::default();
@@ -552,14 +575,6 @@ fn link_staticlib<'a>(
     Ok(())
 }
 
-fn escape_stdout_stderr_string(s: &[u8]) -> String {
-    str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| {
-        let mut x = "Non-UTF-8 output: ".to_string();
-        x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from));
-        x
-    })
-}
-
 /// Use `thorin` (rust implementation of a dwarf packaging utility) to link DWARF objects into a
 /// DWARF package.
 fn link_dwarf_object<'a>(
@@ -764,15 +779,15 @@ fn link_natively<'a>(
                 "Linker does not support -static-pie command line option. Retrying with -static instead."
             );
             // Mirror `add_(pre,post)_link_objects` to replace CRT objects.
-            let self_contained = crt_objects_fallback(sess, crate_type);
+            let self_contained = self_contained(sess, crate_type);
             let opts = &sess.target;
             let pre_objects = if self_contained {
-                &opts.pre_link_objects_fallback
+                &opts.pre_link_objects_self_contained
             } else {
                 &opts.pre_link_objects
             };
             let post_objects = if self_contained {
-                &opts.post_link_objects_fallback
+                &opts.post_link_objects_self_contained
             } else {
                 &opts.post_link_objects
             };
@@ -866,7 +881,7 @@ fn link_natively<'a>(
             if !prog.status.success() {
                 let mut output = prog.stderr.clone();
                 output.extend_from_slice(&prog.stdout);
-                let escaped_output = escape_stdout_stderr_string(&output);
+                let escaped_output = escape_string(&output);
                 let mut err = sess.struct_err(&format!(
                     "linking with `{}` failed: {}",
                     linker_path.display(),
@@ -934,8 +949,8 @@ fn link_natively<'a>(
 
                 sess.abort_if_errors();
             }
-            info!("linker stderr:\n{}", escape_stdout_stderr_string(&prog.stderr));
-            info!("linker stdout:\n{}", escape_stdout_stderr_string(&prog.stdout));
+            info!("linker stderr:\n{}", escape_string(&prog.stderr));
+            info!("linker stdout:\n{}", escape_string(&prog.stdout));
         }
         Err(e) => {
             let linker_not_found = e.kind() == io::ErrorKind::NotFound;
@@ -1065,11 +1080,10 @@ fn strip_symbols_in_osx<'a>(sess: &'a Session, out_filename: &Path, option: Opti
 }
 
 fn escape_string(s: &[u8]) -> String {
-    str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| {
-        let mut x = "Non-UTF-8 output: ".to_string();
-        x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from));
-        x
-    })
+    match str::from_utf8(s) {
+        Ok(s) => s.to_owned(),
+        Err(_) => format!("Non-UTF-8 output: {}", s.escape_ascii()),
+    }
 }
 
 fn add_sanitizer_libraries(sess: &Session, crate_type: CrateType, linker: &mut dyn Linker) {
@@ -1173,13 +1187,6 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
             // only the linker flavor is known; use the default linker for the selected flavor
             (None, Some(flavor)) => Some((
                 PathBuf::from(match flavor {
-                    LinkerFlavor::Em => {
-                        if cfg!(windows) {
-                            "emcc.bat"
-                        } else {
-                            "emcc"
-                        }
-                    }
                     LinkerFlavor::Gcc => {
                         if cfg!(any(target_os = "solaris", target_os = "illumos")) {
                             // On historical Solaris systems, "cc" may have
@@ -1194,11 +1201,17 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
                         }
                     }
                     LinkerFlavor::Ld => "ld",
-                    LinkerFlavor::Msvc => "link.exe",
                     LinkerFlavor::Lld(_) => "lld",
-                    LinkerFlavor::PtxLinker => "rust-ptx-linker",
-                    LinkerFlavor::BpfLinker => "bpf-linker",
-                    LinkerFlavor::L4Bender => "l4-bender",
+                    LinkerFlavor::Msvc => "link.exe",
+                    LinkerFlavor::EmCc => {
+                        if cfg!(windows) {
+                            "emcc.bat"
+                        } else {
+                            "emcc"
+                        }
+                    }
+                    LinkerFlavor::Bpf => "bpf-linker",
+                    LinkerFlavor::Ptx => "rust-ptx-linker",
                 }),
                 flavor,
             )),
@@ -1208,7 +1221,7 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
                 });
 
                 let flavor = if stem == "emcc" {
-                    LinkerFlavor::Em
+                    LinkerFlavor::EmCc
                 } else if stem == "gcc"
                     || stem.ends_with("-gcc")
                     || stem == "clang"
@@ -1236,7 +1249,8 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
 
     // linker and linker flavor specified via command line have precedence over what the target
     // specification specifies
-    if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) {
+    let linker_flavor = sess.opts.cg.linker_flavor.map(LinkerFlavor::from_cli);
+    if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), linker_flavor) {
         return ret;
     }
 
@@ -1556,26 +1570,26 @@ fn detect_self_contained_mingw(sess: &Session) -> bool {
     true
 }
 
-/// Whether we link to our own CRT objects instead of relying on gcc to pull them.
+/// Various toolchain components used during linking are used from rustc distribution
+/// instead of being found somewhere on the host system.
 /// We only provide such support for a very limited number of targets.
-fn crt_objects_fallback(sess: &Session, crate_type: CrateType) -> bool {
+fn self_contained(sess: &Session, crate_type: CrateType) -> bool {
     if let Some(self_contained) = sess.opts.cg.link_self_contained {
         return self_contained;
     }
 
-    match sess.target.crt_objects_fallback {
+    match sess.target.link_self_contained {
+        LinkSelfContainedDefault::False => false,
+        LinkSelfContainedDefault::True => true,
         // FIXME: Find a better heuristic for "native musl toolchain is available",
         // based on host and linker path, for example.
         // (https://github.com/rust-lang/rust/pull/71769#issuecomment-626330237).
-        Some(CrtObjectsFallback::Musl) => sess.crt_static(Some(crate_type)),
-        Some(CrtObjectsFallback::Mingw) => {
+        LinkSelfContainedDefault::Musl => sess.crt_static(Some(crate_type)),
+        LinkSelfContainedDefault::Mingw => {
             sess.host == sess.target
                 && sess.target.vendor != "uwp"
                 && detect_self_contained_mingw(&sess)
         }
-        // FIXME: Figure out cases in which WASM needs to link with a native toolchain.
-        Some(CrtObjectsFallback::Wasm) => true,
-        None => false,
     }
 }
 
@@ -1583,12 +1597,21 @@ fn crt_objects_fallback(sess: &Session, crate_type: CrateType) -> bool {
 fn add_pre_link_objects(
     cmd: &mut dyn Linker,
     sess: &Session,
+    flavor: LinkerFlavor,
     link_output_kind: LinkOutputKind,
     self_contained: bool,
 ) {
+    // FIXME: we are currently missing some infra here (per-linker-flavor CRT objects),
+    // so Fuchsia has to be special-cased.
     let opts = &sess.target;
-    let objects =
-        if self_contained { &opts.pre_link_objects_fallback } else { &opts.pre_link_objects };
+    let empty = Default::default();
+    let objects = if self_contained {
+        &opts.pre_link_objects_self_contained
+    } else if !(sess.target.os == "fuchsia" && flavor == LinkerFlavor::Gcc) {
+        &opts.pre_link_objects
+    } else {
+        &empty
+    };
     for obj in objects.get(&link_output_kind).iter().copied().flatten() {
         cmd.add_object(&get_object_file_path(sess, obj, self_contained));
     }
@@ -1601,9 +1624,11 @@ fn add_post_link_objects(
     link_output_kind: LinkOutputKind,
     self_contained: bool,
 ) {
-    let opts = &sess.target;
-    let objects =
-        if self_contained { &opts.post_link_objects_fallback } else { &opts.post_link_objects };
+    let objects = if self_contained {
+        &sess.target.post_link_objects_self_contained
+    } else {
+        &sess.target.post_link_objects
+    };
     for obj in objects.get(&link_output_kind).iter().copied().flatten() {
         cmd.add_object(&get_object_file_path(sess, obj, self_contained));
     }
@@ -1703,6 +1728,13 @@ fn add_post_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor
 /// that are necessary for the linking. They are only present in symbol table but not actually
 /// used in any sections, so the linker will therefore pick relevant rlibs for linking, but
 /// unused `#[no_mangle]` or `#[used]` can still be discard by GC sections.
+///
+/// There's a few internal crates in the standard library (aka libcore and
+/// libstd) which actually have a circular dependence upon one another. This
+/// currently arises through "weak lang items" where libcore requires things
+/// like `rust_begin_unwind` but libstd ends up defining it. To get this
+/// circular dependence to work correctly we declare some of these things
+/// in this synthetic object.
 fn add_linked_symbol_object(
     cmd: &mut dyn Linker,
     sess: &Session,
@@ -1882,12 +1914,12 @@ fn linker_with_args<'a>(
     out_filename: &Path,
     codegen_results: &CodegenResults,
 ) -> Result<Command, ErrorGuaranteed> {
-    let crt_objects_fallback = crt_objects_fallback(sess, crate_type);
+    let self_contained = self_contained(sess, crate_type);
     let cmd = &mut *super::linker::get_linker(
         sess,
         path,
         flavor,
-        crt_objects_fallback,
+        self_contained,
         &codegen_results.crate_info.target_cpu,
     );
     let link_output_kind = link_output_kind(sess, crate_type);
@@ -1914,7 +1946,7 @@ fn linker_with_args<'a>(
     // ------------ Object code and libraries, order-dependent ------------
 
     // Pre-link CRT objects.
-    add_pre_link_objects(cmd, sess, link_output_kind, crt_objects_fallback);
+    add_pre_link_objects(cmd, sess, flavor, link_output_kind, self_contained);
 
     add_linked_symbol_object(
         cmd,
@@ -1947,7 +1979,6 @@ fn linker_with_args<'a>(
     // Upstream rust libraries are not supposed to depend on our local native
     // libraries as that would violate the structure of the DAG, in that
     // scenario they are required to link to them as well in a shared fashion.
-    // (The current implementation still doesn't prevent it though, see the FIXME below.)
     //
     // Note that upstream rust libraries may contain native dependencies as
     // well, but they also can't depend on what we just started to add to the
@@ -1968,15 +1999,16 @@ fn linker_with_args<'a>(
     // and move this option back to the top.
     cmd.add_as_needed();
 
-    // FIXME: Move this below to other native libraries
-    // (or alternatively link all native libraries after their respective crates).
-    // This change is somewhat breaking in practice due to local static libraries being linked
-    // as whole-archive (#85144), so removing whole-archive may be a pre-requisite.
+    // Local native libraries of all kinds.
+    //
+    // If `-Zlink-native-libraries=false` is set, then the assumption is that an
+    // external build system already has the native dependencies defined, and it
+    // will provide them to the linker itself.
     if sess.opts.unstable_opts.link_native_libraries {
         add_local_native_libraries(cmd, sess, codegen_results);
     }
 
-    // Upstream rust libraries and their non-bundled static libraries
+    // Upstream rust libraries and their (possibly bundled) static native libraries.
     add_upstream_rust_crates(
         cmd,
         sess,
@@ -1986,24 +2018,54 @@ fn linker_with_args<'a>(
         tmpdir,
     );
 
-    // Upstream dynamic native libraries linked with `#[link]` attributes at and `-l`
-    // command line options.
-    // If -Zlink-native-libraries=false is set, then the assumption is that an
-    // external build system already has the native dependencies defined, and it
-    // will provide them to the linker itself.
+    // Dynamic native libraries from upstream crates.
+    //
+    // FIXME: Merge this to `add_upstream_rust_crates` so that all native libraries are linked
+    // together with their respective upstream crates, and in their originally specified order.
+    // This may be slightly breaking due to our use of `--as-needed` and needs a crater run.
     if sess.opts.unstable_opts.link_native_libraries {
         add_upstream_native_libraries(cmd, sess, codegen_results);
     }
 
     // Link with the import library generated for any raw-dylib functions.
     for (raw_dylib_name, raw_dylib_imports) in
-        collate_raw_dylibs(sess, &codegen_results.crate_info.used_libraries)?
+        collate_raw_dylibs(sess, codegen_results.crate_info.used_libraries.iter())?
     {
         cmd.add_object(&archive_builder_builder.create_dll_import_lib(
             sess,
             &raw_dylib_name,
             &raw_dylib_imports,
             tmpdir,
+            true,
+        ));
+    }
+    // As with add_upstream_native_libraries, we need to add the upstream raw-dylib symbols in case
+    // they are used within inlined functions or instantiated generic functions. We do this *after*
+    // handling the raw-dylib symbols in the current crate to make sure that those are chosen first
+    // by the linker.
+    let (_, dependency_linkage) = codegen_results
+        .crate_info
+        .dependency_formats
+        .iter()
+        .find(|(ty, _)| *ty == crate_type)
+        .expect("failed to find crate type in dependency format list");
+    let native_libraries_from_nonstatics = codegen_results
+        .crate_info
+        .native_libraries
+        .iter()
+        .filter_map(|(cnum, libraries)| {
+            (dependency_linkage[cnum.as_usize() - 1] != Linkage::Static).then(|| libraries)
+        })
+        .flatten();
+    for (raw_dylib_name, raw_dylib_imports) in
+        collate_raw_dylibs(sess, native_libraries_from_nonstatics)?
+    {
+        cmd.add_object(&archive_builder_builder.create_dll_import_lib(
+            sess,
+            &raw_dylib_name,
+            &raw_dylib_imports,
+            tmpdir,
+            false,
         ));
     }
 
@@ -2024,7 +2086,7 @@ fn linker_with_args<'a>(
         cmd,
         sess,
         link_output_kind,
-        crt_objects_fallback,
+        self_contained,
         flavor,
         crate_type,
         codegen_results,
@@ -2040,7 +2102,7 @@ fn linker_with_args<'a>(
     // ------------ Object code and libraries, order-dependent ------------
 
     // Post-link CRT objects.
-    add_post_link_objects(cmd, sess, link_output_kind, crt_objects_fallback);
+    add_post_link_objects(cmd, sess, link_output_kind, self_contained);
 
     // ------------ Late order-dependent options ------------
 
@@ -2057,7 +2119,7 @@ fn add_order_independent_options(
     cmd: &mut dyn Linker,
     sess: &Session,
     link_output_kind: LinkOutputKind,
-    crt_objects_fallback: bool,
+    self_contained: bool,
     flavor: LinkerFlavor,
     crate_type: CrateType,
     codegen_results: &CodegenResults,
@@ -2070,7 +2132,10 @@ fn add_order_independent_options(
 
     add_link_script(cmd, sess, tmpdir, crate_type);
 
-    if sess.target.os == "fuchsia" && crate_type == CrateType::Executable {
+    if sess.target.os == "fuchsia"
+        && crate_type == CrateType::Executable
+        && flavor != LinkerFlavor::Gcc
+    {
         let prefix = if sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::ADDRESS) {
             "asan/"
         } else {
@@ -2086,7 +2151,7 @@ fn add_order_independent_options(
     // Make the binary compatible with data execution prevention schemes.
     cmd.add_no_exec();
 
-    if crt_objects_fallback {
+    if self_contained {
         cmd.no_crt_objects();
     }
 
@@ -2099,11 +2164,11 @@ fn add_order_independent_options(
         });
     }
 
-    if flavor == LinkerFlavor::PtxLinker {
+    if flavor == LinkerFlavor::Ptx {
         // Provide the linker with fallback to internal `target-cpu`.
         cmd.arg("--fallback-arch");
         cmd.arg(&codegen_results.crate_info.target_cpu);
-    } else if flavor == LinkerFlavor::BpfLinker {
+    } else if flavor == LinkerFlavor::Bpf {
         cmd.arg("--cpu");
         cmd.arg(&codegen_results.crate_info.target_cpu);
         cmd.arg("--cpu-features");
@@ -2115,7 +2180,7 @@ fn add_order_independent_options(
 
     cmd.linker_plugin_lto();
 
-    add_library_search_dirs(cmd, sess, crt_objects_fallback);
+    add_library_search_dirs(cmd, sess, self_contained);
 
     cmd.output_filename(out_filename);
 
@@ -2319,72 +2384,25 @@ fn add_upstream_rust_crates<'a>(
     // crates.
     let deps = &codegen_results.crate_info.used_crates;
 
-    // There's a few internal crates in the standard library (aka libcore and
-    // libstd) which actually have a circular dependence upon one another. This
-    // currently arises through "weak lang items" where libcore requires things
-    // like `rust_begin_unwind` but libstd ends up defining it. To get this
-    // circular dependence to work correctly in all situations we'll need to be
-    // sure to correctly apply the `--start-group` and `--end-group` options to
-    // GNU linkers, otherwise if we don't use any other symbol from the standard
-    // library it'll get discarded and the whole application won't link.
-    //
-    // In this loop we're calculating the `group_end`, after which crate to
-    // pass `--end-group` and `group_start`, before which crate to pass
-    // `--start-group`. We currently do this by passing `--end-group` after
-    // the first crate (when iterating backwards) that requires a lang item
-    // defined somewhere else. Once that's set then when we've defined all the
-    // necessary lang items we'll pass `--start-group`.
-    //
-    // Note that this isn't amazing logic for now but it should do the trick
-    // for the current implementation of the standard library.
-    let mut group_end = None;
-    let mut group_start = None;
-    // Crates available for linking thus far.
-    let mut available = FxHashSet::default();
-    // Crates required to satisfy dependencies discovered so far.
-    let mut required = FxHashSet::default();
-
-    let info = &codegen_results.crate_info;
-    for &cnum in deps.iter().rev() {
-        if let Some(missing) = info.missing_lang_items.get(&cnum) {
-            let missing_crates = missing.iter().map(|i| info.lang_item_to_crate.get(i).copied());
-            required.extend(missing_crates);
-        }
-
-        required.insert(Some(cnum));
-        available.insert(Some(cnum));
-
-        if required.len() > available.len() && group_end.is_none() {
-            group_end = Some(cnum);
-        }
-        if required.len() == available.len() && group_end.is_some() {
-            group_start = Some(cnum);
-            break;
-        }
-    }
-
-    // If we didn't end up filling in all lang items from upstream crates then
-    // we'll be filling it in with our crate. This probably means we're the
-    // standard library itself, so skip this for now.
-    if group_end.is_some() && group_start.is_none() {
-        group_end = None;
-    }
-
     let mut compiler_builtins = None;
     let search_path = OnceCell::new();
 
     for &cnum in deps.iter() {
-        if group_start == Some(cnum) {
-            cmd.group_start();
-        }
-
         // We may not pass all crates through to the linker. Some crates may
         // appear statically in an existing dylib, meaning we'll pick up all the
         // symbols from the dylib.
         let src = &codegen_results.crate_info.used_crate_source[&cnum];
         match data[cnum.as_usize() - 1] {
             _ if codegen_results.crate_info.profiler_runtime == Some(cnum) => {
-                add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
+                add_static_crate(
+                    cmd,
+                    sess,
+                    archive_builder_builder,
+                    codegen_results,
+                    tmpdir,
+                    cnum,
+                    &Default::default(),
+                );
             }
             // compiler-builtins are always placed last to ensure that they're
             // linked correctly.
@@ -2394,17 +2412,41 @@ fn add_upstream_rust_crates<'a>(
             }
             Linkage::NotLinked | Linkage::IncludedFromDylib => {}
             Linkage::Static => {
-                add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
+                let bundled_libs = if sess.opts.unstable_opts.packed_bundled_libs {
+                    codegen_results.crate_info.native_libraries[&cnum]
+                        .iter()
+                        .filter_map(|lib| lib.filename)
+                        .collect::<FxHashSet<_>>()
+                } else {
+                    Default::default()
+                };
+                add_static_crate(
+                    cmd,
+                    sess,
+                    archive_builder_builder,
+                    codegen_results,
+                    tmpdir,
+                    cnum,
+                    &bundled_libs,
+                );
 
                 // Link static native libs with "-bundle" modifier only if the crate they originate from
                 // is being linked statically to the current crate.  If it's linked dynamically
                 // or is an rlib already included via some other dylib crate, the symbols from
                 // native libs will have already been included in that dylib.
                 //
-                // If -Zlink-native-libraries=false is set, then the assumption is that an
+                // If `-Zlink-native-libraries=false` is set, then the assumption is that an
                 // external build system already has the native dependencies defined, and it
                 // will provide them to the linker itself.
                 if sess.opts.unstable_opts.link_native_libraries {
+                    if sess.opts.unstable_opts.packed_bundled_libs {
+                        // If rlib contains native libs as archives, unpack them to tmpdir.
+                        let rlib = &src.rlib.as_ref().unwrap().0;
+                        archive_builder_builder
+                            .extract_bundled_libs(rlib, tmpdir, &bundled_libs)
+                            .unwrap_or_else(|e| sess.fatal(e));
+                    }
+
                     let mut last = (None, NativeLibKind::Unspecified, None);
                     for lib in &codegen_results.crate_info.native_libraries[&cnum] {
                         let Some(name) = lib.name else {
@@ -2437,6 +2479,14 @@ fn add_upstream_rust_crates<'a>(
                                 bundle: Some(false),
                                 whole_archive: Some(false) | None,
                             } => {
+                                // HACK/FIXME: Fixup a circular dependency between libgcc and libc
+                                // with glibc. This logic should be moved to the libc crate.
+                                if sess.target.os == "linux"
+                                    && sess.target.env == "gnu"
+                                    && name == "c"
+                                {
+                                    cmd.link_staticlib("gcc", false);
+                                }
                                 cmd.link_staticlib(name, lib.verbatim.unwrap_or(false));
                             }
                             NativeLibKind::LinkArg => {
@@ -2446,20 +2496,23 @@ fn add_upstream_rust_crates<'a>(
                             | NativeLibKind::Framework { .. }
                             | NativeLibKind::Unspecified
                             | NativeLibKind::RawDylib => {}
-                            NativeLibKind::Static {
-                                bundle: Some(true) | None,
-                                whole_archive: _,
-                            } => {}
+                            NativeLibKind::Static { bundle: Some(true) | None, whole_archive } => {
+                                if sess.opts.unstable_opts.packed_bundled_libs {
+                                    // If rlib contains native libs as archives, they are unpacked to tmpdir.
+                                    let path = tmpdir.join(lib.filename.unwrap().as_str());
+                                    if whole_archive == Some(true) {
+                                        cmd.link_whole_rlib(&path);
+                                    } else {
+                                        cmd.link_rlib(&path);
+                                    }
+                                }
+                            }
                         }
                     }
                 }
             }
             Linkage::Dynamic => add_dynamic_crate(cmd, sess, &src.dylib.as_ref().unwrap().0),
         }
-
-        if group_end == Some(cnum) {
-            cmd.group_end();
-        }
     }
 
     // compiler-builtins are always placed last to ensure that they're
@@ -2468,7 +2521,15 @@ fn add_upstream_rust_crates<'a>(
     // was already "included" in a dylib (e.g., `libstd` when `-C prefer-dynamic`
     // is used)
     if let Some(cnum) = compiler_builtins {
-        add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
+        add_static_crate(
+            cmd,
+            sess,
+            archive_builder_builder,
+            codegen_results,
+            tmpdir,
+            cnum,
+            &Default::default(),
+        );
     }
 
     // Converts a library file-stem into a cc -l argument
@@ -2501,6 +2562,7 @@ fn add_upstream_rust_crates<'a>(
         codegen_results: &CodegenResults,
         tmpdir: &Path,
         cnum: CrateNum,
+        bundled_lib_file_names: &FxHashSet<Symbol>,
     ) {
         let src = &codegen_results.crate_info.used_crate_source[&cnum];
         let cratepath = &src.rlib.as_ref().unwrap().0;
@@ -2529,6 +2591,7 @@ fn add_upstream_rust_crates<'a>(
         let dst = tmpdir.join(cratepath.file_name().unwrap());
         let name = cratepath.file_name().unwrap().to_str().unwrap();
         let name = &name[3..name.len() - 5]; // chop off lib/.rlib
+        let bundled_lib_file_names = bundled_lib_file_names.clone();
 
         sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| {
             let canonical_name = name.replace('-', "_");
@@ -2562,6 +2625,15 @@ fn add_upstream_rust_crates<'a>(
                     let skip_because_lto =
                         upstream_rust_objects_already_included && is_rust_object && is_builtins;
 
+                    // We skip native libraries because:
+                    // 1. This native libraries won't be used from the generated rlib,
+                    //    so we can throw them away to avoid the copying work.
+                    // 2. We can't allow it to be a single remaining entry in archive
+                    //    as some linkers may complain on that.
+                    if bundled_lib_file_names.contains(&Symbol::intern(f)) {
+                        return true;
+                    }
+
                     if skip_because_cfg_say_so || skip_because_lto {
                         return true;
                     }
@@ -2657,7 +2729,7 @@ fn relevant_lib(sess: &Session, lib: &NativeLib) -> bool {
     }
 }
 
-fn are_upstream_rust_objects_already_included(sess: &Session) -> bool {
+pub(crate) fn are_upstream_rust_objects_already_included(sess: &Session) -> bool {
     match sess.lto() {
         config::Lto::Fat => true,
         config::Lto::Thin => {
@@ -2674,11 +2746,16 @@ fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
     let os = &sess.target.os;
     let llvm_target = &sess.target.llvm_target;
     if sess.target.vendor != "apple"
-        || !matches!(os.as_ref(), "ios" | "tvos" | "watchos")
+        || !matches!(os.as_ref(), "ios" | "tvos" | "watchos" | "macos")
         || (flavor != LinkerFlavor::Gcc && flavor != LinkerFlavor::Lld(LldFlavor::Ld64))
     {
         return;
     }
+
+    if os == "macos" && flavor != LinkerFlavor::Lld(LldFlavor::Ld64) {
+        return;
+    }
+
     let sdk_name = match (arch.as_ref(), os.as_ref()) {
         ("aarch64", "tvos") => "appletvos",
         ("x86_64", "tvos") => "appletvsimulator",
@@ -2694,6 +2771,7 @@ fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
         ("aarch64", "watchos") if llvm_target.ends_with("-simulator") => "watchsimulator",
         ("aarch64", "watchos") => "watchos",
         ("arm", "watchos") => "watchos",
+        (_, "macos") => "macosx",
         _ => {
             sess.err(&format!("unsupported arch `{}` for os `{}`", arch, os));
             return;
@@ -2777,20 +2855,24 @@ fn add_gcc_ld_path(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
         if let LinkerFlavor::Gcc = flavor {
             match ld_impl {
                 LdImpl::Lld => {
-                    let tools_path = sess.get_tools_search_paths(false);
-                    let gcc_ld_dir = tools_path
-                        .into_iter()
-                        .map(|p| p.join("gcc-ld"))
-                        .find(|p| {
-                            p.join(if sess.host.is_like_windows { "ld.exe" } else { "ld" }).exists()
-                        })
-                        .unwrap_or_else(|| sess.fatal("rust-lld (as ld) not found"));
-                    cmd.arg({
-                        let mut arg = OsString::from("-B");
-                        arg.push(gcc_ld_dir);
-                        arg
-                    });
-                    cmd.arg(format!("-Wl,-rustc-lld-flavor={}", sess.target.lld_flavor.as_str()));
+                    // Implement the "self-contained" part of -Zgcc-ld
+                    // by adding rustc distribution directories to the tool search path.
+                    for path in sess.get_tools_search_paths(false) {
+                        cmd.arg({
+                            let mut arg = OsString::from("-B");
+                            arg.push(path.join("gcc-ld"));
+                            arg
+                        });
+                    }
+                    // Implement the "linker flavor" part of -Zgcc-ld
+                    // by asking cc to use some kind of lld.
+                    cmd.arg("-fuse-ld=lld");
+                    if sess.target.lld_flavor != LldFlavor::Ld {
+                        // Tell clang to use a non-default LLD flavor.
+                        // Gcc doesn't understand the target option, but we currently assume
+                        // that gcc is not used for Apple and Wasm targets (#97402).
+                        cmd.arg(format!("--target={}", sess.target.llvm_target));
+                    }
                 }
             }
         } else {
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index ce51b2e95..e505543b2 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -1,4 +1,3 @@
-use super::archive;
 use super::command::Command;
 use super::symbol_export;
 use rustc_span::symbol::sym;
@@ -11,6 +10,7 @@ use std::path::{Path, PathBuf};
 use std::{env, mem, str};
 
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
+use rustc_metadata::find_native_static_library;
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind};
 use rustc_middle::ty::TyCtxt;
@@ -126,29 +126,26 @@ pub fn get_linker<'a>(
     // to the linker args construction.
     assert!(cmd.get_args().is_empty() || sess.target.vendor == "uwp");
     match flavor {
-        LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => {
-            Box::new(MsvcLinker { cmd, sess }) as Box<dyn Linker>
-        }
-        LinkerFlavor::Em => Box::new(EmLinker { cmd, sess }) as Box<dyn Linker>,
         LinkerFlavor::Gcc => {
             Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: false })
                 as Box<dyn Linker>
         }
-
+        LinkerFlavor::Ld if sess.target.os == "l4re" => {
+            Box::new(L4Bender::new(cmd, sess)) as Box<dyn Linker>
+        }
         LinkerFlavor::Lld(LldFlavor::Ld)
         | LinkerFlavor::Lld(LldFlavor::Ld64)
         | LinkerFlavor::Ld => {
             Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: true })
                 as Box<dyn Linker>
         }
-
+        LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => {
+            Box::new(MsvcLinker { cmd, sess }) as Box<dyn Linker>
+        }
         LinkerFlavor::Lld(LldFlavor::Wasm) => Box::new(WasmLd::new(cmd, sess)) as Box<dyn Linker>,
-
-        LinkerFlavor::PtxLinker => Box::new(PtxLinker { cmd, sess }) as Box<dyn Linker>,
-
-        LinkerFlavor::BpfLinker => Box::new(BpfLinker { cmd, sess }) as Box<dyn Linker>,
-
-        LinkerFlavor::L4Bender => Box::new(L4Bender::new(cmd, sess)) as Box<dyn Linker>,
+        LinkerFlavor::EmCc => Box::new(EmLinker { cmd, sess }) as Box<dyn Linker>,
+        LinkerFlavor::Bpf => Box::new(BpfLinker { cmd, sess }) as Box<dyn Linker>,
+        LinkerFlavor::Ptx => Box::new(PtxLinker { cmd, sess }) as Box<dyn Linker>,
     }
 }
 
@@ -186,8 +183,6 @@ pub trait Linker {
     fn no_default_libraries(&mut self);
     fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]);
     fn subsystem(&mut self, subsystem: &str);
-    fn group_start(&mut self);
-    fn group_end(&mut self);
     fn linker_plugin_lto(&mut self);
     fn add_eh_frame_header(&mut self) {}
     fn add_no_exec(&mut self) {}
@@ -519,7 +514,7 @@ impl<'a> Linker for GccLinker<'a> {
             // -force_load is the macOS equivalent of --whole-archive, but it
             // involves passing the full path to the library to link.
             self.linker_arg("-force_load");
-            let lib = archive::find_library(lib, verbatim, search_path, &self.sess);
+            let lib = find_native_static_library(lib, Some(verbatim), search_path, &self.sess);
             self.linker_arg(&lib);
         }
     }
@@ -733,18 +728,6 @@ impl<'a> Linker for GccLinker<'a> {
         self.hint_dynamic(); // Reset to default before returning the composed command line.
     }
 
-    fn group_start(&mut self) {
-        if self.takes_hints() {
-            self.linker_arg("--start-group");
-        }
-    }
-
-    fn group_end(&mut self) {
-        if self.takes_hints() {
-            self.linker_arg("--end-group");
-        }
-    }
-
     fn linker_plugin_lto(&mut self) {
         match self.sess.opts.cg.linker_plugin_lto {
             LinkerPluginLto::Disabled => {
@@ -1022,10 +1005,6 @@ impl<'a> Linker for MsvcLinker<'a> {
         }
     }
 
-    // MSVC doesn't need group indicators
-    fn group_start(&mut self) {}
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {
         // Do nothing
     }
@@ -1168,10 +1147,6 @@ impl<'a> Linker for EmLinker<'a> {
         // noop
     }
 
-    // Appears not necessary on Emscripten
-    fn group_start(&mut self) {}
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {
         // Do nothing
     }
@@ -1347,10 +1322,6 @@ impl<'a> Linker for WasmLd<'a> {
 
     fn subsystem(&mut self, _subsystem: &str) {}
 
-    // Not needed for now with LLD
-    fn group_start(&mut self) {}
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {
         // Do nothing for now
     }
@@ -1479,14 +1450,6 @@ impl<'a> Linker for L4Bender<'a> {
         self.hint_static(); // Reset to default before returning the composed command line.
     }
 
-    fn group_start(&mut self) {
-        self.cmd.arg("--start-group");
-    }
-
-    fn group_end(&mut self) {
-        self.cmd.arg("--end-group");
-    }
-
     fn linker_plugin_lto(&mut self) {}
 
     fn control_flow_guard(&mut self) {}
@@ -1667,10 +1630,6 @@ impl<'a> Linker for PtxLinker<'a> {
 
     fn subsystem(&mut self, _subsystem: &str) {}
 
-    fn group_start(&mut self) {}
-
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {}
 }
 
@@ -1780,9 +1739,5 @@ impl<'a> Linker for BpfLinker<'a> {
 
     fn subsystem(&mut self, _subsystem: &str) {}
 
-    fn group_start(&mut self) {}
-
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {}
 }
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index 0302c2881..99ddd1764 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -117,6 +117,10 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
         "riscv32" => Architecture::Riscv32,
         "riscv64" => Architecture::Riscv64,
         "sparc64" => Architecture::Sparc64,
+        "avr" => Architecture::Avr,
+        "msp430" => Architecture::Msp430,
+        "hexagon" => Architecture::Hexagon,
+        "bpf" => Architecture::Bpf,
         // Unsupported architecture.
         _ => return None,
     };
@@ -187,12 +191,12 @@ pub enum MetadataPosition {
     Last,
 }
 
-// For rlibs we "pack" rustc metadata into a dummy object file. When rustc
-// creates a dylib crate type it will pass `--whole-archive` (or the
-// platform equivalent) to include all object files from an rlib into the
-// final dylib itself. This causes linkers to iterate and try to include all
-// files located in an archive, so if metadata is stored in an archive then
-// it needs to be of a form that the linker will be able to process.
+// For rlibs we "pack" rustc metadata into a dummy object file.
+//
+// Historically it was needed because rustc linked rlibs as whole-archive in some cases.
+// In that case linkers try to include all files located in an archive, so if metadata is stored
+// in an archive then it needs to be of a form that the linker is able to process.
+// Now it's not clear whether metadata still needs to be wrapped into an object file or not.
 //
 // Note, though, that we don't actually want this metadata to show up in any
 // final output of the compiler. Instead this is purely for rustc's own
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index e6b605575..8d7e2c5cf 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -103,18 +103,14 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
             }
         })
         .map(|def_id| {
-            let (export_level, used) = if special_runtime_crate {
-                let name = tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())).name;
-                // We won't link right if these symbols are stripped during LTO.
-                let used = match name {
-                    "rust_eh_personality"
-                    | "rust_eh_register_frames"
-                    | "rust_eh_unregister_frames" => true,
-                    _ => false,
-                };
-                (SymbolExportLevel::Rust, used)
+            // We won't link right if this symbol is stripped during LTO.
+            let name = tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())).name;
+            let used = name == "rust_eh_personality";
+
+            let export_level = if special_runtime_crate {
+                SymbolExportLevel::Rust
             } else {
-                (symbol_export_level(tcx, def_id.to_def_id()), false)
+                symbol_export_level(tcx, def_id.to_def_id())
             };
             let codegen_attrs = tcx.codegen_fn_attrs(def_id.to_def_id());
             debug!(
@@ -544,7 +540,7 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
         .map(|fnabi| (fnabi.conv, &fnabi.args[..]))
         .unwrap_or((Conv::Rust, &[]));
 
-    // Decorate symbols with prefices, suffices and total number of bytes of arguments.
+    // Decorate symbols with prefixes, suffixes and total number of bytes of arguments.
     // Reference: https://docs.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170
     let (prefix, suffix) = match conv {
         Conv::X86Fastcall => ("@", "@"),
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 1b5ad8710..68f3b19b7 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -15,7 +15,7 @@ use rustc_data_structures::profiling::TimingGuard;
 use rustc_data_structures::profiling::VerboseTimingGuard;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::emitter::Emitter;
-use rustc_errors::{DiagnosticId, FatalError, Handler, Level};
+use rustc_errors::{translation::Translate, DiagnosticId, FatalError, Handler, Level};
 use rustc_fs_util::link_or_copy;
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_incremental::{
@@ -256,8 +256,11 @@ impl ModuleConfig {
             {
                 MergeFunctions::Disabled => false,
                 MergeFunctions::Trampolines | MergeFunctions::Aliases => {
-                    sess.opts.optimize == config::OptLevel::Default
-                        || sess.opts.optimize == config::OptLevel::Aggressive
+                    use config::OptLevel::*;
+                    match sess.opts.optimize {
+                        Aggressive | Default | SizeMin | Size => true,
+                        Less | No => false,
+                    }
                 }
             },
 
@@ -1737,6 +1740,16 @@ impl SharedEmitter {
     }
 }
 
+impl Translate for SharedEmitter {
+    fn fluent_bundle(&self) -> Option<&Lrc<rustc_errors::FluentBundle>> {
+        None
+    }
+
+    fn fallback_fluent_bundle(&self) -> &rustc_errors::FluentBundle {
+        panic!("shared emitter attempted to translate a diagnostic");
+    }
+}
+
 impl Emitter for SharedEmitter {
     fn emit_diagnostic(&mut self, diag: &rustc_errors::Diagnostic) {
         let fluent_args = self.to_fluent_args(diag.args());
@@ -1758,14 +1771,6 @@ impl Emitter for SharedEmitter {
     fn source_map(&self) -> Option<&Lrc<SourceMap>> {
         None
     }
-
-    fn fluent_bundle(&self) -> Option<&Lrc<rustc_errors::FluentBundle>> {
-        None
-    }
-
-    fn fallback_fluent_bundle(&self) -> &rustc_errors::FluentBundle {
-        panic!("shared emitter attempted to translate a diagnostic");
-    }
 }
 
 impl SharedEmitterMain {
@@ -1887,7 +1892,7 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> {
             }
         });
 
-        sess.cgu_reuse_tracker.check_expected_reuse(sess.diagnostic());
+        sess.cgu_reuse_tracker.check_expected_reuse(sess);
 
         sess.abort_if_errors();
 
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index a840b2709..b98ff4957 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -1,3 +1,4 @@
+use crate::back::link::are_upstream_rust_objects_already_included;
 use crate::back::metadata::create_compressed_metadata_file;
 use crate::back::write::{
     compute_per_cgu_lto_type, start_async_codegen, submit_codegened_module_to_llvm,
@@ -12,7 +13,7 @@ use crate::traits::*;
 use crate::{CachedModuleCodegen, CompiledModule, CrateInfo, MemFlags, ModuleCodegen, ModuleKind};
 
 use rustc_attr as attr;
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
 
 use rustc_data_structures::sync::par_iter;
@@ -21,10 +22,12 @@ use rustc_data_structures::sync::ParallelIterator;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_hir::lang_items::LangItem;
+use rustc_hir::weak_lang_items::WEAK_ITEMS_SYMBOLS;
 use rustc_index::vec::Idx;
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::middle::exported_symbols;
+use rustc_middle::middle::exported_symbols::SymbolExportKind;
 use rustc_middle::middle::lang_items;
 use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
@@ -34,6 +37,7 @@ use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{self, CrateType, EntryFnType, OutputType};
 use rustc_session::Session;
 use rustc_span::symbol::sym;
+use rustc_span::Symbol;
 use rustc_span::{DebuggerVisualizerFile, DebuggerVisualizerType};
 use rustc_target::abi::{Align, VariantIdx};
 
@@ -151,6 +155,7 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             let old_info =
                 old_info.expect("unsized_info: missing old info for trait upcasting coercion");
             if data_a.principal_def_id() == data_b.principal_def_id() {
+                // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables.
                 return old_info;
             }
 
@@ -162,6 +167,11 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             if let Some(entry_idx) = vptr_entry_idx {
                 let ptr_ty = cx.type_i8p();
                 let ptr_align = cx.tcx().data_layout.pointer_align.abi;
+                let vtable_ptr_ty = cx.scalar_pair_element_backend_type(
+                    cx.layout_of(cx.tcx().mk_mut_ptr(target)),
+                    1,
+                    true,
+                );
                 let llvtable = bx.pointercast(old_info, bx.type_ptr_to(ptr_ty));
                 let gep = bx.inbounds_gep(
                     ptr_ty,
@@ -172,7 +182,7 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 bx.nonnull_metadata(new_vptr);
                 // VTable loads are invariant.
                 bx.set_invariant_load(new_vptr);
-                new_vptr
+                bx.pointercast(new_vptr, vtable_ptr_ty)
             } else {
                 old_info
             }
@@ -388,15 +398,14 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
     let main_llfn = cx.get_fn_addr(instance);
 
-    let use_start_lang_item = EntryFnType::Start != entry_type;
-    let entry_fn = create_entry_fn::<Bx>(cx, main_llfn, main_def_id, use_start_lang_item);
+    let entry_fn = create_entry_fn::<Bx>(cx, main_llfn, main_def_id, entry_type);
     return Some(entry_fn);
 
     fn create_entry_fn<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         cx: &'a Bx::CodegenCx,
         rust_main: Bx::Value,
         rust_main_def_id: DefId,
-        use_start_lang_item: bool,
+        entry_type: EntryFnType,
     ) -> Bx::Function {
         // The entry function is either `int main(void)` or `int main(int argc, char **argv)`,
         // depending on whether the target needs `argc` and `argv` to be passed in.
@@ -441,7 +450,7 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         let i8pp_ty = cx.type_ptr_to(cx.type_i8p());
         let (arg_argc, arg_argv) = get_argc_argv(cx, &mut bx);
 
-        let (start_fn, start_ty, args) = if use_start_lang_item {
+        let (start_fn, start_ty, args) = if let EntryFnType::Main { sigpipe } = entry_type {
             let start_def_id = cx.tcx().require_lang_item(LangItem::Start, None);
             let start_fn = cx.get_fn_addr(
                 ty::Instance::resolve(
@@ -453,8 +462,13 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 .unwrap()
                 .unwrap(),
             );
-            let start_ty = cx.type_func(&[cx.val_ty(rust_main), isize_ty, i8pp_ty], isize_ty);
-            (start_fn, start_ty, vec![rust_main, arg_argc, arg_argv])
+
+            let i8_ty = cx.type_i8();
+            let arg_sigpipe = bx.const_u8(sigpipe);
+
+            let start_ty =
+                cx.type_func(&[cx.val_ty(rust_main), isize_ty, i8pp_ty, i8_ty], isize_ty);
+            (start_fn, start_ty, vec![rust_main, arg_argc, arg_argv, arg_sigpipe])
         } else {
             debug!("using user-defined start fn");
             let start_ty = cx.type_func(&[isize_ty, i8pp_ty], isize_ty);
@@ -810,21 +824,16 @@ impl CrateInfo {
             crate_name: Default::default(),
             used_crates,
             used_crate_source: Default::default(),
-            lang_item_to_crate: Default::default(),
-            missing_lang_items: Default::default(),
             dependency_formats: tcx.dependency_formats(()).clone(),
             windows_subsystem,
             natvis_debugger_visualizers: Default::default(),
         };
-        let lang_items = tcx.lang_items();
-
         let crates = tcx.crates(());
 
         let n_crates = crates.len();
         info.native_libraries.reserve(n_crates);
         info.crate_name.reserve(n_crates);
         info.used_crate_source.reserve(n_crates);
-        info.missing_lang_items.reserve(n_crates);
 
         for &cnum in crates.iter() {
             info.native_libraries
@@ -842,17 +851,41 @@ impl CrateInfo {
             if tcx.is_no_builtins(cnum) {
                 info.is_no_builtins.insert(cnum);
             }
-            let missing = tcx.missing_lang_items(cnum);
-            for &item in missing.iter() {
-                if let Ok(id) = lang_items.require(item) {
-                    info.lang_item_to_crate.insert(item, id.krate);
-                }
-            }
+        }
 
-            // No need to look for lang items that don't actually need to exist.
-            let missing =
-                missing.iter().cloned().filter(|&l| lang_items::required(tcx, l)).collect();
-            info.missing_lang_items.insert(cnum, missing);
+        // Handle circular dependencies in the standard library.
+        // See comment before `add_linked_symbol_object` function for the details.
+        // If global LTO is enabled then almost everything (*) is glued into a single object file,
+        // so this logic is not necessary and can cause issues on some targets (due to weak lang
+        // item symbols being "privatized" to that object file), so we disable it.
+        // (*) Native libs, and `#[compiler_builtins]` and `#[no_builtins]` crates are not glued,
+        // and we assume that they cannot define weak lang items. This is not currently enforced
+        // by the compiler, but that's ok because all this stuff is unstable anyway.
+        let target = &tcx.sess.target;
+        if !are_upstream_rust_objects_already_included(tcx.sess) {
+            let missing_weak_lang_items: FxHashSet<&Symbol> = info
+                .used_crates
+                .iter()
+                .flat_map(|cnum| {
+                    tcx.missing_lang_items(*cnum)
+                        .iter()
+                        .filter(|l| lang_items::required(tcx, **l))
+                        .filter_map(|item| WEAK_ITEMS_SYMBOLS.get(item))
+                })
+                .collect();
+            let prefix = if target.is_like_windows && target.arch == "x86" { "_" } else { "" };
+            info.linked_symbols
+                .iter_mut()
+                .filter(|(crate_type, _)| {
+                    !matches!(crate_type, CrateType::Rlib | CrateType::Staticlib)
+                })
+                .for_each(|(_, linked_symbols)| {
+                    linked_symbols.extend(
+                        missing_weak_lang_items
+                            .iter()
+                            .map(|item| (format!("{prefix}{item}"), SymbolExportKind::Text)),
+                    )
+                });
         }
 
         let embed_visualizers = tcx.sess.crate_types().iter().any(|&crate_type| match crate_type {
@@ -873,7 +906,7 @@ impl CrateInfo {
             }
         });
 
-        if tcx.sess.target.is_like_msvc && embed_visualizers {
+        if target.is_like_msvc && embed_visualizers {
             info.natvis_debugger_visualizers =
                 collect_debugger_visualizers_transitive(tcx, DebuggerVisualizerType::Natvis);
         }
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 8cd5a0fc2..135ed680d 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -18,11 +18,10 @@ use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathD
 use rustc_hir::{AsyncGeneratorKind, GeneratorKind, Mutability};
 use rustc_middle::ty::layout::{IntegerExt, TyAndLayout};
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
-use rustc_middle::ty::{self, ExistentialProjection, GeneratorSubsts, ParamEnv, Ty, TyCtxt};
-use rustc_target::abi::{Integer, TagEncoding, Variants};
+use rustc_middle::ty::{self, ExistentialProjection, ParamEnv, Ty, TyCtxt};
+use rustc_target::abi::Integer;
 use smallvec::SmallVec;
 
-use std::borrow::Cow;
 use std::fmt::Write;
 
 use crate::debuginfo::wants_c_like_enum_debuginfo;
@@ -98,7 +97,6 @@ fn push_debuginfo_type_name<'tcx>(
 
             if let Some(ty_and_layout) = layout_for_cpp_like_fallback {
                 msvc_enum_fallback(
-                    tcx,
                     ty_and_layout,
                     &|output, visited| {
                         push_item_name(tcx, def.did(), true, output);
@@ -391,11 +389,10 @@ fn push_debuginfo_type_name<'tcx>(
             // Name will be "{closure_env#0}<T1, T2, ...>", "{generator_env#0}<T1, T2, ...>", or
             // "{async_fn_env#0}<T1, T2, ...>", etc.
             // In the case of cpp-like debuginfo, the name additionally gets wrapped inside of
-            // an artificial `enum$<>` type, as defined in msvc_enum_fallback().
+            // an artificial `enum2$<>` type, as defined in msvc_enum_fallback().
             if cpp_like_debuginfo && t.is_generator() {
                 let ty_and_layout = tcx.layout_of(ParamEnv::reveal_all().and(t)).unwrap();
                 msvc_enum_fallback(
-                    tcx,
                     ty_and_layout,
                     &|output, visited| {
                         push_closure_or_generator_name(tcx, def_id, substs, true, output, visited);
@@ -428,58 +425,17 @@ fn push_debuginfo_type_name<'tcx>(
 
     /// MSVC names enums differently than other platforms so that the debugging visualization
     // format (natvis) is able to understand enums and render the active variant correctly in the
-    // debugger. For more information, look in `src/etc/natvis/intrinsic.natvis` and
-    // `EnumMemberDescriptionFactor::create_member_descriptions`.
+    // debugger. For more information, look in
+    // rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs.
     fn msvc_enum_fallback<'tcx>(
-        tcx: TyCtxt<'tcx>,
         ty_and_layout: TyAndLayout<'tcx>,
         push_inner: &dyn Fn(/*output*/ &mut String, /*visited*/ &mut FxHashSet<Ty<'tcx>>),
         output: &mut String,
         visited: &mut FxHashSet<Ty<'tcx>>,
     ) {
         debug_assert!(!wants_c_like_enum_debuginfo(ty_and_layout));
-        let ty = ty_and_layout.ty;
-
-        output.push_str("enum$<");
+        output.push_str("enum2$<");
         push_inner(output, visited);
-
-        let variant_name = |variant_index| match ty.kind() {
-            ty::Adt(adt_def, _) => {
-                debug_assert!(adt_def.is_enum());
-                Cow::from(adt_def.variant(variant_index).name.as_str())
-            }
-            ty::Generator(..) => GeneratorSubsts::variant_name(variant_index),
-            _ => unreachable!(),
-        };
-
-        if let Variants::Multiple {
-            tag_encoding: TagEncoding::Niche { dataful_variant, .. },
-            tag,
-            variants,
-            ..
-        } = &ty_and_layout.variants
-        {
-            let dataful_variant_layout = &variants[*dataful_variant];
-
-            // calculate the range of values for the dataful variant
-            let dataful_discriminant_range =
-                dataful_variant_layout.largest_niche().unwrap().valid_range;
-
-            let min = dataful_discriminant_range.start;
-            let min = tag.size(&tcx).truncate(min);
-
-            let max = dataful_discriminant_range.end;
-            let max = tag.size(&tcx).truncate(max);
-
-            let dataful_variant_name = variant_name(*dataful_variant);
-            write!(output, ", {}, {}, {}", min, max, dataful_variant_name).unwrap();
-        } else if let Variants::Single { index: variant_idx } = &ty_and_layout.variants {
-            // Uninhabited enums can't be constructed and should never need to be visualized so
-            // skip this step for them.
-            if !ty_and_layout.abi.is_uninhabited() {
-                write!(output, ", {}", variant_name(*variant_idx)).unwrap();
-            }
-        }
         push_close_angle_bracket(true, output);
     }
 
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 1802eedf1..e736b2aba 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -1,7 +1,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(box_patterns)]
 #![feature(try_blocks)]
-#![feature(let_else)]
+#![cfg_attr(bootstrap, feature(let_else))]
 #![feature(once_cell)]
 #![feature(associated_type_bounds)]
 #![feature(strict_provenance)]
@@ -25,7 +25,6 @@ use rustc_ast as ast;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lrc;
 use rustc_hir::def_id::CrateNum;
-use rustc_hir::LangItem;
 use rustc_middle::dep_graph::WorkProduct;
 use rustc_middle::middle::dependency_format::Dependencies;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
@@ -113,6 +112,7 @@ bitflags::bitflags! {
 pub struct NativeLib {
     pub kind: NativeLibKind,
     pub name: Option<Symbol>,
+    pub filename: Option<Symbol>,
     pub cfg: Option<ast::MetaItem>,
     pub verbatim: Option<bool>,
     pub dll_imports: Vec<cstore::DllImport>,
@@ -122,6 +122,7 @@ impl From<&cstore::NativeLib> for NativeLib {
     fn from(lib: &cstore::NativeLib) -> Self {
         NativeLib {
             kind: lib.kind,
+            filename: lib.filename,
             name: lib.name,
             cfg: lib.cfg.clone(),
             verbatim: lib.verbatim,
@@ -152,8 +153,6 @@ pub struct CrateInfo {
     pub used_libraries: Vec<NativeLib>,
     pub used_crate_source: FxHashMap<CrateNum, Lrc<CrateSource>>,
     pub used_crates: Vec<CrateNum>,
-    pub lang_item_to_crate: FxHashMap<LangItem, CrateNum>,
-    pub missing_lang_items: FxHashMap<CrateNum, Vec<LangItem>>,
     pub dependency_formats: Lrc<Dependencies>,
     pub windows_subsystem: Option<String>,
     pub natvis_debugger_visualizers: BTreeSet<DebuggerVisualizerFile>,
@@ -168,6 +167,13 @@ pub struct CodegenResults {
     pub crate_info: CrateInfo,
 }
 
+pub enum CodegenErrors<'a> {
+    WrongFileType,
+    EmptyVersionNumber,
+    EncodingVersionMismatch { version_array: String, rlink_version: u32 },
+    RustcVersionMismatch { rustc_version: String, current_version: &'a str },
+}
+
 pub fn provide(providers: &mut Providers) {
     crate::back::symbol_export::provide(providers);
     crate::base::provide(providers);
@@ -212,30 +218,34 @@ impl CodegenResults {
         encoder.finish()
     }
 
-    pub fn deserialize_rlink(data: Vec<u8>) -> Result<Self, String> {
+    pub fn deserialize_rlink<'a>(data: Vec<u8>) -> Result<Self, CodegenErrors<'a>> {
         // The Decodable machinery is not used here because it panics if the input data is invalid
         // and because its internal representation may change.
         if !data.starts_with(RLINK_MAGIC) {
-            return Err("The input does not look like a .rlink file".to_string());
+            return Err(CodegenErrors::WrongFileType);
         }
         let data = &data[RLINK_MAGIC.len()..];
         if data.len() < 4 {
-            return Err("The input does not contain version number".to_string());
+            return Err(CodegenErrors::EmptyVersionNumber);
         }
 
         let mut version_array: [u8; 4] = Default::default();
         version_array.copy_from_slice(&data[..4]);
         if u32::from_be_bytes(version_array) != RLINK_VERSION {
-            return Err(".rlink file was produced with encoding version {version_array}, but the current version is {RLINK_VERSION}".to_string());
+            return Err(CodegenErrors::EncodingVersionMismatch {
+                version_array: String::from_utf8_lossy(&version_array).to_string(),
+                rlink_version: RLINK_VERSION,
+            });
         }
 
         let mut decoder = MemDecoder::new(&data[4..], 0);
         let rustc_version = decoder.read_str();
         let current_version = RUSTC_VERSION.unwrap();
         if rustc_version != current_version {
-            return Err(format!(
-                ".rlink file was produced by rustc version {rustc_version}, but the current version is {current_version}."
-            ));
+            return Err(CodegenErrors::RustcVersionMismatch {
+                rustc_version: rustc_version.to_string(),
+                current_version,
+            });
         }
 
         let codegen_results = CodegenResults::decode(&mut decoder);
diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs
index 27d791d90..cae46ebd2 100644
--- a/compiler/rustc_codegen_ssa/src/meth.rs
+++ b/compiler/rustc_codegen_ssa/src/meth.rs
@@ -1,6 +1,6 @@
 use crate::traits::*;
 
-use rustc_middle::ty::{self, subst::GenericArgKind, ExistentialPredicate, Ty, TyCtxt};
+use rustc_middle::ty::{self, subst::GenericArgKind, Ty};
 use rustc_session::config::Lto;
 use rustc_symbol_mangling::typeid_for_trait_ref;
 use rustc_target::abi::call::FnAbi;
@@ -29,7 +29,7 @@ impl<'a, 'tcx> VirtualIndex {
             && bx.cx().sess().lto() == Lto::Fat
         {
             let typeid =
-                bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), get_trait_ref(bx.tcx(), ty)));
+                bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), expect_dyn_trait_in_self(ty)));
             let vtable_byte_offset = self.0 * bx.data_layout().pointer_size.bytes();
             let type_checked_load = bx.type_checked_load(llvtable, vtable_byte_offset, typeid);
             let func = bx.extract_value(type_checked_load, 0);
@@ -64,17 +64,13 @@ impl<'a, 'tcx> VirtualIndex {
     }
 }
 
-fn get_trait_ref<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> {
+/// This takes a valid `self` receiver type and extracts the principal trait
+/// ref of the type.
+fn expect_dyn_trait_in_self<'tcx>(ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> {
     for arg in ty.peel_refs().walk() {
         if let GenericArgKind::Type(ty) = arg.unpack() {
-            if let ty::Dynamic(trait_refs, _) = ty.kind() {
-                return trait_refs[0].map_bound(|trait_ref| match trait_ref {
-                    ExistentialPredicate::Trait(tr) => tr,
-                    ExistentialPredicate::Projection(proj) => proj.trait_ref(tcx),
-                    ExistentialPredicate::AutoTrait(_) => {
-                        bug!("auto traits don't have functions")
-                    }
-                });
+            if let ty::Dynamic(data, _, _) = ty.kind() {
+                return data.principal().expect("expected principal trait object");
             }
         }
     }
@@ -90,6 +86,7 @@ fn get_trait_ref<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::PolyExistentialTr
 /// The `trait_ref` encodes the erased self type. Hence if we are
 /// making an object `Foo<dyn Trait>` from a value of type `Foo<T>`, then
 /// `trait_ref` would map `T: Trait`.
+#[instrument(level = "debug", skip(cx))]
 pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
     cx: &Cx,
     ty: Ty<'tcx>,
@@ -97,8 +94,6 @@ pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
 ) -> Cx::Value {
     let tcx = cx.tcx();
 
-    debug!("get_vtable(ty={:?}, trait_ref={:?})", ty, trait_ref);
-
     // Check the cache.
     if let Some(&val) = cx.vtables().borrow().get(&(ty, trait_ref)) {
         return val;
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index 24da48ead..c7617d2e4 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -266,7 +266,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
         result: &mut IndexVec<mir::BasicBlock, CleanupKind>,
         mir: &mir::Body<'tcx>,
     ) {
-        for (bb, data) in mir.basic_blocks().iter_enumerated() {
+        for (bb, data) in mir.basic_blocks.iter_enumerated() {
             match data.terminator().kind {
                 TerminatorKind::Goto { .. }
                 | TerminatorKind::Resume
@@ -296,7 +296,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
     }
 
     fn propagate<'tcx>(result: &mut IndexVec<mir::BasicBlock, CleanupKind>, mir: &mir::Body<'tcx>) {
-        let mut funclet_succs = IndexVec::from_elem(None, mir.basic_blocks());
+        let mut funclet_succs = IndexVec::from_elem(None, &mir.basic_blocks);
 
         let mut set_successor = |funclet: mir::BasicBlock, succ| match funclet_succs[funclet] {
             ref mut s @ None => {
@@ -359,7 +359,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
         }
     }
 
-    let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, mir.basic_blocks());
+    let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, &mir.basic_blocks);
 
     discover_masters(&mut result, mir);
     propagate(&mut result, mir);
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 3eee58d9d..a6b226ef7 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -13,15 +13,14 @@ use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::lang_items::LangItem;
 use rustc_index::vec::Idx;
-use rustc_middle::mir::AssertKind;
-use rustc_middle::mir::{self, SwitchTargets};
+use rustc_middle::mir::{self, AssertKind, SwitchTargets};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty, TypeVisitable};
 use rustc_span::source_map::Span;
 use rustc_span::{sym, Symbol};
 use rustc_symbol_mangling::typeid::typeid_for_fnabi;
-use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
+use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode, Reg};
 use rustc_target::abi::{self, HasDataLayout, WrappingRange};
 use rustc_target::spec::abi::Abi;
 
@@ -324,7 +323,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             bx.unreachable();
             return;
         }
-        let llval = match self.fn_abi.ret.mode {
+        let llval = match &self.fn_abi.ret.mode {
             PassMode::Ignore | PassMode::Indirect { .. } => {
                 bx.ret_void();
                 return;
@@ -339,7 +338,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            PassMode::Cast(cast_ty) => {
+            PassMode::Cast(cast_ty, _) => {
                 let op = match self.locals[mir::RETURN_PLACE] {
                     LocalRef::Operand(Some(op)) => op,
                     LocalRef::Operand(None) => bug!("use of return before def"),
@@ -360,7 +359,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         llval
                     }
                 };
-                let ty = bx.cast_backend_type(&cast_ty);
+                let ty = bx.cast_backend_type(cast_ty);
                 let addr = bx.pointercast(llslot, bx.type_ptr_to(ty));
                 bx.load(ty, addr, self.fn_abi.ret.layout.align.abi)
             }
@@ -368,6 +367,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         bx.ret(llval);
     }
 
+    #[tracing::instrument(level = "trace", skip(self, helper, bx))]
     fn codegen_drop_terminator(
         &mut self,
         helper: TerminatorCodegenHelper<'tcx>,
@@ -398,13 +398,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let (drop_fn, fn_abi) = match ty.kind() {
             // FIXME(eddyb) perhaps move some of this logic into
             // `Instance::resolve_drop_in_place`?
-            ty::Dynamic(..) => {
+            ty::Dynamic(_, _, ty::Dyn) => {
+                // IN THIS ARM, WE HAVE:
+                // ty = *mut (dyn Trait)
+                // which is: exists<T> ( *mut T,    Vtable<T: Trait> )
+                //                       args[0]    args[1]
+                //
+                // args = ( Data, Vtable )
+                //                  |
+                //                  v
+                //                /-------\
+                //                | ...   |
+                //                \-------/
+                //
                 let virtual_drop = Instance {
                     def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
                     substs: drop_fn.substs,
                 };
+                debug!("ty = {:?}", ty);
+                debug!("drop_fn = {:?}", drop_fn);
+                debug!("args = {:?}", args);
                 let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
                 let vtable = args[1];
+                // Truncate vtable off of args list
                 args = &args[..1];
                 (
                     meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
@@ -412,6 +428,51 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     fn_abi,
                 )
             }
+            ty::Dynamic(_, _, ty::DynStar) => {
+                // IN THIS ARM, WE HAVE:
+                // ty = *mut (dyn* Trait)
+                // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
+                //
+                // args = [ * ]
+                //          |
+                //          v
+                //      ( Data, Vtable )
+                //                |
+                //                v
+                //              /-------\
+                //              | ...   |
+                //              \-------/
+                //
+                //
+                // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
+                //
+                // data = &(*args[0]).0    // gives a pointer to Data above (really the same pointer)
+                // vtable = (*args[0]).1   // loads the vtable out
+                // (data, vtable)          // an equivalent Rust `*mut dyn Trait`
+                //
+                // SO THEN WE CAN USE THE ABOVE CODE.
+                let virtual_drop = Instance {
+                    def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+                    substs: drop_fn.substs,
+                };
+                debug!("ty = {:?}", ty);
+                debug!("drop_fn = {:?}", drop_fn);
+                debug!("args = {:?}", args);
+                let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
+                let data = args[0];
+                let data_ty = bx.cx().backend_type(place.layout);
+                let vtable_ptr =
+                    bx.gep(data_ty, data, &[bx.cx().const_i32(0), bx.cx().const_i32(1)]);
+                let vtable = bx.load(bx.type_i8p(), vtable_ptr, abi::Align::ONE);
+                // Truncate vtable off of args list
+                args = &args[..1];
+                debug!("args' = {:?}", args);
+                (
+                    meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
+                        .get_fn(&mut bx, vtable, ty, &fn_abi),
+                    fn_abi,
+                )
+            }
             _ => (bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty())),
         };
         helper.do_call(
@@ -798,58 +859,78 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             let mut op = self.codegen_operand(&mut bx, arg);
 
             if let (0, Some(ty::InstanceDef::Virtual(_, idx))) = (i, def) {
-                if let Pair(..) = op.val {
-                    // In the case of Rc<Self>, we need to explicitly pass a
-                    // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
-                    // that is understood elsewhere in the compiler as a method on
-                    // `dyn Trait`.
-                    // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
-                    // we get a value of a built-in pointer type
-                    'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
-                        && !op.layout.ty.is_region_ptr()
-                    {
-                        for i in 0..op.layout.fields.count() {
-                            let field = op.extract_field(&mut bx, i);
-                            if !field.layout.is_zst() {
-                                // we found the one non-zero-sized field that is allowed
-                                // now find *its* non-zero-sized field, or stop if it's a
-                                // pointer
-                                op = field;
-                                continue 'descend_newtypes;
+                match op.val {
+                    Pair(data_ptr, meta) => {
+                        // In the case of Rc<Self>, we need to explicitly pass a
+                        // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
+                        // that is understood elsewhere in the compiler as a method on
+                        // `dyn Trait`.
+                        // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
+                        // we get a value of a built-in pointer type
+                        'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
+                            && !op.layout.ty.is_region_ptr()
+                        {
+                            for i in 0..op.layout.fields.count() {
+                                let field = op.extract_field(&mut bx, i);
+                                if !field.layout.is_zst() {
+                                    // we found the one non-zero-sized field that is allowed
+                                    // now find *its* non-zero-sized field, or stop if it's a
+                                    // pointer
+                                    op = field;
+                                    continue 'descend_newtypes;
+                                }
                             }
+
+                            span_bug!(span, "receiver has no non-zero-sized fields {:?}", op);
                         }
 
-                        span_bug!(span, "receiver has no non-zero-sized fields {:?}", op);
+                        // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
+                        // data pointer and vtable. Look up the method in the vtable, and pass
+                        // the data pointer as the first argument
+                        llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+                            &mut bx,
+                            meta,
+                            op.layout.ty,
+                            &fn_abi,
+                        ));
+                        llargs.push(data_ptr);
+                        continue 'make_args;
                     }
-
-                    // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
-                    // data pointer and vtable. Look up the method in the vtable, and pass
-                    // the data pointer as the first argument
-                    match op.val {
-                        Pair(data_ptr, meta) => {
-                            llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
-                                &mut bx,
-                                meta,
-                                op.layout.ty,
-                                &fn_abi,
-                            ));
-                            llargs.push(data_ptr);
-                            continue 'make_args;
+                    Ref(data_ptr, Some(meta), _) => {
+                        // by-value dynamic dispatch
+                        llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+                            &mut bx,
+                            meta,
+                            op.layout.ty,
+                            &fn_abi,
+                        ));
+                        llargs.push(data_ptr);
+                        continue;
+                    }
+                    Immediate(_) => {
+                        let ty::Ref(_, ty, _) = op.layout.ty.kind() else {
+                            span_bug!(span, "can't codegen a virtual call on {:#?}", op);
+                        };
+                        if !ty.is_dyn_star() {
+                            span_bug!(span, "can't codegen a virtual call on {:#?}", op);
                         }
-                        other => bug!("expected a Pair, got {:?}", other),
+                        // FIXME(dyn-star): Make sure this is done on a &dyn* receiver
+                        let place = op.deref(bx.cx());
+                        let data_ptr = place.project_field(&mut bx, 0);
+                        let meta_ptr = place.project_field(&mut bx, 1);
+                        let meta = bx.load_operand(meta_ptr);
+                        llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+                            &mut bx,
+                            meta.immediate(),
+                            op.layout.ty,
+                            &fn_abi,
+                        ));
+                        llargs.push(data_ptr.llval);
+                        continue;
+                    }
+                    _ => {
+                        span_bug!(span, "can't codegen a virtual call on {:#?}", op);
                     }
-                } else if let Ref(data_ptr, Some(meta), _) = op.val {
-                    // by-value dynamic dispatch
-                    llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
-                        &mut bx,
-                        meta,
-                        op.layout.ty,
-                        &fn_abi,
-                    ));
-                    llargs.push(data_ptr);
-                    continue;
-                } else {
-                    span_bug!(span, "can't codegen a virtual call on {:?}", op);
                 }
             }
 
@@ -1161,39 +1242,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         llargs: &mut Vec<Bx::Value>,
         arg: &ArgAbi<'tcx, Ty<'tcx>>,
     ) {
-        // Fill padding with undef value, where applicable.
-        if let Some(ty) = arg.pad {
-            llargs.push(bx.const_undef(bx.reg_backend_type(&ty)))
-        }
-
-        if arg.is_ignore() {
-            return;
-        }
-
-        if let PassMode::Pair(..) = arg.mode {
-            match op.val {
+        match arg.mode {
+            PassMode::Ignore => return,
+            PassMode::Cast(_, true) => {
+                // Fill padding with undef value, where applicable.
+                llargs.push(bx.const_undef(bx.reg_backend_type(&Reg::i32())));
+            }
+            PassMode::Pair(..) => match op.val {
                 Pair(a, b) => {
                     llargs.push(a);
                     llargs.push(b);
                     return;
                 }
                 _ => bug!("codegen_argument: {:?} invalid for pair argument", op),
-            }
-        } else if arg.is_unsized_indirect() {
-            match op.val {
+            },
+            PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => match op.val {
                 Ref(a, Some(b), _) => {
                     llargs.push(a);
                     llargs.push(b);
                     return;
                 }
                 _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
-            }
+            },
+            _ => {}
         }
 
         // Force by-ref if we have to load through a cast pointer.
         let (mut llval, align, by_ref) = match op.val {
             Immediate(_) | Pair(..) => match arg.mode {
-                PassMode::Indirect { .. } | PassMode::Cast(_) => {
+                PassMode::Indirect { .. } | PassMode::Cast(..) => {
                     let scratch = PlaceRef::alloca(bx, arg.layout);
                     op.val.store(bx, scratch);
                     (scratch.llval, scratch.align, true)
@@ -1225,8 +1302,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         if by_ref && !arg.is_indirect() {
             // Have to load the argument, maybe while casting it.
-            if let PassMode::Cast(ty) = arg.mode {
-                let llty = bx.cast_backend_type(&ty);
+            if let PassMode::Cast(ty, _) = &arg.mode {
+                let llty = bx.cast_backend_type(ty);
                 let addr = bx.pointercast(llval, bx.type_ptr_to(llty));
                 llval = bx.load(llty, addr, align.min(arg.layout.align.abi));
             } else {
@@ -1625,7 +1702,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
             DirectOperand(index) => {
                 // If there is a cast, we have to store and reload.
-                let op = if let PassMode::Cast(_) = ret_abi.mode {
+                let op = if let PassMode::Cast(..) = ret_abi.mode {
                     let tmp = PlaceRef::alloca(bx, ret_abi.layout);
                     tmp.storage_live(bx);
                     bx.store_arg(&ret_abi, llval, tmp);
diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs
index 9a995fbf6..4c6ab457c 100644
--- a/compiler/rustc_codegen_ssa/src/mir/constant.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs
@@ -25,26 +25,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         constant: &mir::Constant<'tcx>,
     ) -> Result<ConstValue<'tcx>, ErrorHandled> {
         let ct = self.monomorphize(constant.literal);
-        let ct = match ct {
-            mir::ConstantKind::Ty(ct) => ct,
+        let uv = match ct {
+            mir::ConstantKind::Ty(ct) => match ct.kind() {
+                ty::ConstKind::Unevaluated(uv) => uv.expand(),
+                ty::ConstKind::Value(val) => {
+                    return Ok(self.cx.tcx().valtree_to_const_val((ct.ty(), val)));
+                }
+                err => span_bug!(
+                    constant.span,
+                    "encountered bad ConstKind after monomorphizing: {:?}",
+                    err
+                ),
+            },
+            mir::ConstantKind::Unevaluated(uv, _) => uv,
             mir::ConstantKind::Val(val, _) => return Ok(val),
         };
-        match ct.kind() {
-            ty::ConstKind::Unevaluated(ct) => self
-                .cx
-                .tcx()
-                .const_eval_resolve(ty::ParamEnv::reveal_all(), ct, None)
-                .map_err(|err| {
-                    self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered");
-                    err
-                }),
-            ty::ConstKind::Value(val) => Ok(self.cx.tcx().valtree_to_const_val((ct.ty(), val))),
-            err => span_bug!(
-                constant.span,
-                "encountered bad ConstKind after monomorphizing: {:?}",
-                err
-            ),
-        }
+
+        self.cx.tcx().const_eval_resolve(ty::ParamEnv::reveal_all(), uv, None).map_err(|err| {
+            self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered");
+            err
+        })
     }
 
     /// process constant containing SIMD shuffle indices
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
index 8c3186efc..157c1c823 100644
--- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -3,7 +3,7 @@ use rustc_index::vec::IndexVec;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir;
 use rustc_middle::ty;
-use rustc_middle::ty::layout::LayoutOf;
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 use rustc_session::config::DebugInfo;
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{BytePos, Span};
@@ -93,15 +93,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     }
 
     /// In order to have a good line stepping behavior in debugger, we overwrite debug
-    /// locations of macro expansions with that of the outermost expansion site
-    /// (unless the crate is being compiled with `-Z debug-macros`).
+    /// locations of macro expansions with that of the outermost expansion site (when the macro is
+    /// annotated with `#[collapse_debuginfo]` or when `-Zdebug-macros` is provided).
     fn adjust_span_for_debugging(&self, mut span: Span) -> Span {
         // Bail out if debug info emission is not enabled.
         if self.debug_context.is_none() {
             return span;
         }
 
-        if span.from_expansion() && !self.cx.sess().opts.unstable_opts.debug_macros {
+        if self.cx.tcx().should_collapse_debuginfo(span) {
             // Walk up the macro expansion chain until we reach a non-expanded span.
             // We also stop at the function body level because no line stepping can occur
             // at the level above that.
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 94ac71a4d..215edbe02 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -77,10 +77,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
 
         let llval = match name {
-            sym::assume => {
-                bx.assume(args[0].immediate());
-                return;
-            }
             sym::abort => {
                 bx.abort();
                 return;
@@ -555,14 +551,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 return;
             }
 
-            sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => {
+            sym::ptr_guaranteed_cmp => {
                 let a = args[0].immediate();
                 let b = args[1].immediate();
-                if name == sym::ptr_guaranteed_eq {
-                    bx.icmp(IntPredicate::IntEQ, a, b)
-                } else {
-                    bx.icmp(IntPredicate::IntNE, a, b)
-                }
+                bx.icmp(IntPredicate::IntEQ, a, b)
             }
 
             sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
@@ -597,8 +589,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         };
 
         if !fn_abi.ret.is_ignore() {
-            if let PassMode::Cast(ty) = fn_abi.ret.mode {
-                let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(&ty));
+            if let PassMode::Cast(ty, _) = &fn_abi.ret.mode {
+                let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(ty));
                 let ptr = bx.pointercast(result.llval, ptr_llty);
                 bx.store(llval, ptr, result.align);
             } else {
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index 8ee375fa9..2b931bfc9 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -150,13 +150,13 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     let start_llbb = Bx::append_block(cx, llfn, "start");
     let mut bx = Bx::build(cx, start_llbb);
 
-    if mir.basic_blocks().iter().any(|bb| bb.is_cleanup) {
+    if mir.basic_blocks.iter().any(|bb| bb.is_cleanup) {
         bx.set_personality_fn(cx.eh_personality());
     }
 
     let cleanup_kinds = analyze::cleanup_kinds(&mir);
     let cached_llbbs: IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>> = mir
-        .basic_blocks()
+        .basic_blocks
         .indices()
         .map(|bb| if bb == mir::START_BLOCK { Some(start_llbb) } else { None })
         .collect();
@@ -172,8 +172,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         unreachable_block: None,
         double_unwind_guard: None,
         cleanup_kinds,
-        landing_pads: IndexVec::from_elem(None, mir.basic_blocks()),
-        funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks().len()),
+        landing_pads: IndexVec::from_elem(None, &mir.basic_blocks),
+        funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()),
         locals: IndexVec::new(),
         debug_context,
         per_local_var_debug_info: None,
@@ -191,7 +191,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 // errored or at least linted
                 ErrorHandled::Reported(_) | ErrorHandled::Linted => {}
                 ErrorHandled::TooGeneric => {
-                    span_bug!(const_.span, "codgen encountered polymorphic constant: {:?}", err)
+                    span_bug!(const_.span, "codegen encountered polymorphic constant: {:?}", err)
                 }
             }
         }
@@ -283,7 +283,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 for i in 0..tupled_arg_tys.len() {
                     let arg = &fx.fn_abi.args[idx];
                     idx += 1;
-                    if arg.pad.is_some() {
+                    if let PassMode::Cast(_, true) = arg.mode {
                         llarg_idx += 1;
                     }
                     let pr_field = place.project_field(bx, i);
@@ -309,7 +309,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
             let arg = &fx.fn_abi.args[idx];
             idx += 1;
-            if arg.pad.is_some() {
+            if let PassMode::Cast(_, true) = arg.mode {
                 llarg_idx += 1;
             }
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index c612634fc..37b1e0362 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -72,10 +72,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
     ) -> Self {
         let layout = bx.layout_of(ty);
 
-        if layout.is_zst() {
-            return OperandRef::new_zst(bx, layout);
-        }
-
         let val = match val {
             ConstValue::Scalar(x) => {
                 let Abi::Scalar(scalar) = layout.abi else {
@@ -84,10 +80,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
                 OperandValue::Immediate(llval)
             }
-            ConstValue::ZeroSized => {
-                let llval = bx.zst_to_backend(bx.immediate_backend_type(layout));
-                OperandValue::Immediate(llval)
-            }
+            ConstValue::ZeroSized => return OperandRef::new_zst(bx, layout),
             ConstValue::Slice { data, start, end } => {
                 let Abi::ScalarPair(a_scalar, _) = layout.abi else {
                     bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index 268c4d765..13d8f6edd 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -4,7 +4,6 @@ use super::{FunctionCx, LocalRef};
 use crate::common::IntPredicate;
 use crate::glue;
 use crate::traits::*;
-use crate::MemFlags;
 
 use rustc_middle::mir;
 use rustc_middle::mir::tcx::PlaceTy;
@@ -245,7 +244,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                 };
                 bx.intcast(tag.immediate(), cast_to, signed)
             }
-            TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
+            TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
                 // Rebase from niche values to discriminants, and check
                 // whether the result is in range for the niche variants.
                 let niche_llty = bx.cx().immediate_backend_type(tag.layout);
@@ -303,7 +302,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                 bx.select(
                     is_niche,
                     niche_discr,
-                    bx.cx().const_uint(cast_to, dataful_variant.as_u32() as u64),
+                    bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64),
                 )
             }
         }
@@ -338,21 +337,11 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
             }
             Variants::Multiple {
                 tag_encoding:
-                    TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
+                    TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
                 tag_field,
                 ..
             } => {
-                if variant_index != dataful_variant {
-                    if bx.cx().sess().target.arch == "arm"
-                        || bx.cx().sess().target.arch == "aarch64"
-                    {
-                        // FIXME(#34427): as workaround for LLVM bug on ARM,
-                        // use memset of 0 before assigning niche value.
-                        let fill_byte = bx.cx().const_u8(0);
-                        let size = bx.cx().const_usize(self.layout.size.bytes());
-                        bx.memset(self.llval, fill_byte, size, self.align, MemFlags::empty());
-                    }
-
+                if variant_index != untagged_variant {
                     let niche = self.project_field(bx, tag_field);
                     let niche_llty = bx.cx().immediate_backend_type(niche.layout);
                     let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 26b9fbf44..56852b0fc 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -4,6 +4,7 @@ use super::{FunctionCx, LocalRef};
 
 use crate::base;
 use crate::common::{self, IntPredicate};
+use crate::meth::get_vtable;
 use crate::traits::*;
 use crate::MemFlags;
 
@@ -87,7 +88,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     let size = bx.const_usize(dest.layout.size.bytes());
 
                     // Use llvm.memset.p0i8.* to initialize all zero arrays
-                    if bx.cx().const_to_opt_uint(v) == Some(0) {
+                    if bx.cx().const_to_opt_u128(v, false) == Some(0) {
                         let fill = bx.cx().const_u8(0);
                         bx.memset(start, fill, size, dest.align, MemFlags::empty());
                         return bx;
@@ -271,6 +272,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             bug!("unexpected non-pair operand");
                         }
                     }
+                    mir::CastKind::DynStar => {
+                        let data = match operand.val {
+                            OperandValue::Ref(_, _, _) => todo!(),
+                            OperandValue::Immediate(v) => v,
+                            OperandValue::Pair(_, _) => todo!(),
+                        };
+                        let trait_ref =
+                            if let ty::Dynamic(data, _, ty::DynStar) = cast.ty.kind() {
+                                data.principal()
+                            } else {
+                                bug!("Only valid to do a DynStar cast into a DynStar type")
+                            };
+                        let vtable = get_vtable(bx.cx(), source.ty(self.mir, bx.tcx()), trait_ref);
+                        OperandValue::Pair(data, vtable)
+                    }
                     mir::CastKind::Pointer(
                         PointerCast::MutToConstPointer | PointerCast::ArrayToPointer,
                     )
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
index f452f2988..1db0fb3a6 100644
--- a/compiler/rustc_codegen_ssa/src/mir/statement.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -1,4 +1,5 @@
 use rustc_middle::mir;
+use rustc_middle::mir::NonDivergingIntrinsic;
 
 use super::FunctionCx;
 use super::LocalRef;
@@ -73,11 +74,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope);
                 bx
             }
-            mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
-                ref src,
-                ref dst,
-                ref count,
-            }) => {
+            mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
+                let op_val = self.codegen_operand(&mut bx, op);
+                bx.assume(op_val.immediate());
+                bx
+            }
+            mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
+                mir::CopyNonOverlapping { ref count, ref src, ref dst },
+            )) => {
                 let dst_val = self.codegen_operand(&mut bx, dst);
                 let src_val = self.codegen_operand(&mut bx, src);
                 let count = self.codegen_operand(&mut bx, count).immediate();
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index ecad05185..0e259bcd1 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -210,8 +210,11 @@ const POWERPC_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("vsx", Some(sym::powerpc_target_feature)),
 ];
 
-const MIPS_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] =
-    &[("fp64", Some(sym::mips_target_feature)), ("msa", Some(sym::mips_target_feature))];
+const MIPS_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
+    ("fp64", Some(sym::mips_target_feature)),
+    ("msa", Some(sym::mips_target_feature)),
+    ("virt", Some(sym::mips_target_feature)),
+];
 
 const RISCV_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("m", Some(sym::riscv_target_feature)),
@@ -227,6 +230,10 @@ const RISCV_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("zhinxmin", Some(sym::riscv_target_feature)),
     ("zfh", Some(sym::riscv_target_feature)),
     ("zfhmin", Some(sym::riscv_target_feature)),
+    ("zba", Some(sym::riscv_target_feature)),
+    ("zbb", Some(sym::riscv_target_feature)),
+    ("zbc", Some(sym::riscv_target_feature)),
+    ("zbs", Some(sym::riscv_target_feature)),
     ("zbkb", Some(sym::riscv_target_feature)),
     ("zbkc", Some(sym::riscv_target_feature)),
     ("zbkx", Some(sym::riscv_target_feature)),
diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs
index 9f49749bb..10cf8948b 100644
--- a/compiler/rustc_codegen_ssa/src/traits/builder.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs
@@ -1,6 +1,5 @@
 use super::abi::AbiBuilderMethods;
 use super::asm::AsmBuilderMethods;
-use super::consts::ConstMethods;
 use super::coverageinfo::CoverageInfoBuilderMethods;
 use super::debuginfo::DebugInfoBuilderMethods;
 use super::intrinsic::IntrinsicCallMethods;
@@ -15,7 +14,6 @@ use crate::mir::operand::OperandRef;
 use crate::mir::place::PlaceRef;
 use crate::MemFlags;
 
-use rustc_apfloat::{ieee, Float, Round, Status};
 use rustc_middle::ty::layout::{HasParamEnv, TyAndLayout};
 use rustc_middle::ty::Ty;
 use rustc_span::Span;
@@ -188,8 +186,8 @@ pub trait BuilderMethods<'a, 'tcx>:
 
     fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
-    fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
-    fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
+    fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+    fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
@@ -223,156 +221,7 @@ pub trait BuilderMethods<'a, 'tcx>:
             return if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) };
         }
 
-        let try_sat_result =
-            if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) };
-        if let Some(try_sat_result) = try_sat_result {
-            return try_sat_result;
-        }
-
-        let int_width = self.cx().int_width(int_ty);
-        let float_width = self.cx().float_width(float_ty);
-        // LLVM's fpto[su]i returns undef when the input x is infinite, NaN, or does not fit into the
-        // destination integer type after rounding towards zero. This `undef` value can cause UB in
-        // safe code (see issue #10184), so we implement a saturating conversion on top of it:
-        // Semantically, the mathematical value of the input is rounded towards zero to the next
-        // mathematical integer, and then the result is clamped into the range of the destination
-        // integer type. Positive and negative infinity are mapped to the maximum and minimum value of
-        // the destination integer type. NaN is mapped to 0.
-        //
-        // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to
-        // a value representable in int_ty.
-        // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits.
-        // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two.
-        // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly
-        // representable. Note that this only works if float_ty's exponent range is sufficiently large.
-        // f16 or 256 bit integers would break this property. Right now the smallest float type is f32
-        // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127.
-        // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because
-        // we're rounding towards zero, we just get float_ty::MAX (which is always an integer).
-        // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX.
-        let int_max = |signed: bool, int_width: u64| -> u128 {
-            let shift_amount = 128 - int_width;
-            if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount }
-        };
-        let int_min = |signed: bool, int_width: u64| -> i128 {
-            if signed { i128::MIN >> (128 - int_width) } else { 0 }
-        };
-
-        let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) {
-            let rounded_min =
-                ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero);
-            assert_eq!(rounded_min.status, Status::OK);
-            let rounded_max =
-                ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero);
-            assert!(rounded_max.value.is_finite());
-            (rounded_min.value.to_bits(), rounded_max.value.to_bits())
-        };
-        let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) {
-            let rounded_min =
-                ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero);
-            assert_eq!(rounded_min.status, Status::OK);
-            let rounded_max =
-                ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero);
-            assert!(rounded_max.value.is_finite());
-            (rounded_min.value.to_bits(), rounded_max.value.to_bits())
-        };
-        // To implement saturation, we perform the following steps:
-        //
-        // 1. Cast x to an integer with fpto[su]i. This may result in undef.
-        // 2. Compare x to f_min and f_max, and use the comparison results to select:
-        //  a) int_ty::MIN if x < f_min or x is NaN
-        //  b) int_ty::MAX if x > f_max
-        //  c) the result of fpto[su]i otherwise
-        // 3. If x is NaN, return 0.0, otherwise return the result of step 2.
-        //
-        // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the
-        // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of
-        // undef does not introduce any non-determinism either.
-        // More importantly, the above procedure correctly implements saturating conversion.
-        // Proof (sketch):
-        // If x is NaN, 0 is returned by definition.
-        // Otherwise, x is finite or infinite and thus can be compared with f_min and f_max.
-        // This yields three cases to consider:
-        // (1) if x in [f_min, f_max], the result of fpto[su]i is returned, which agrees with
-        //     saturating conversion for inputs in that range.
-        // (2) if x > f_max, then x is larger than int_ty::MAX. This holds even if f_max is rounded
-        //     (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger
-        //     than int_ty::MAX. Because x is larger than int_ty::MAX, the return value of int_ty::MAX
-        //     is correct.
-        // (3) if x < f_min, then x is smaller than int_ty::MIN. As shown earlier, f_min exactly equals
-        //     int_ty::MIN and therefore the return value of int_ty::MIN is correct.
-        // QED.
-
-        let float_bits_to_llval = |bx: &mut Self, bits| {
-            let bits_llval = match float_width {
-                32 => bx.cx().const_u32(bits as u32),
-                64 => bx.cx().const_u64(bits as u64),
-                n => bug!("unsupported float width {}", n),
-            };
-            bx.bitcast(bits_llval, float_ty)
-        };
-        let (f_min, f_max) = match float_width {
-            32 => compute_clamp_bounds_single(signed, int_width),
-            64 => compute_clamp_bounds_double(signed, int_width),
-            n => bug!("unsupported float width {}", n),
-        };
-        let f_min = float_bits_to_llval(self, f_min);
-        let f_max = float_bits_to_llval(self, f_max);
-        let int_max = self.cx().const_uint_big(int_ty, int_max(signed, int_width));
-        let int_min = self.cx().const_uint_big(int_ty, int_min(signed, int_width) as u128);
-        let zero = self.cx().const_uint(int_ty, 0);
-
-        // If we're working with vectors, constants must be "splatted": the constant is duplicated
-        // into each lane of the vector.  The algorithm stays the same, we are just using the
-        // same constant across all lanes.
-        let maybe_splat = |bx: &mut Self, val| {
-            if bx.cx().type_kind(dest_ty) == TypeKind::Vector {
-                bx.vector_splat(bx.vector_length(dest_ty), val)
-            } else {
-                val
-            }
-        };
-        let f_min = maybe_splat(self, f_min);
-        let f_max = maybe_splat(self, f_max);
-        let int_max = maybe_splat(self, int_max);
-        let int_min = maybe_splat(self, int_min);
-        let zero = maybe_splat(self, zero);
-
-        // Step 1 ...
-        let fptosui_result = if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) };
-        let less_or_nan = self.fcmp(RealPredicate::RealULT, x, f_min);
-        let greater = self.fcmp(RealPredicate::RealOGT, x, f_max);
-
-        // Step 2: We use two comparisons and two selects, with %s1 being the
-        // result:
-        //     %less_or_nan = fcmp ult %x, %f_min
-        //     %greater = fcmp olt %x, %f_max
-        //     %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result
-        //     %s1 = select %greater, int_ty::MAX, %s0
-        // Note that %less_or_nan uses an *unordered* comparison. This
-        // comparison is true if the operands are not comparable (i.e., if x is
-        // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if
-        // x is NaN.
-        //
-        // Performance note: Unordered comparison can be lowered to a "flipped"
-        // comparison and a negation, and the negation can be merged into the
-        // select. Therefore, it not necessarily any more expensive than an
-        // ordered ("normal") comparison. Whether these optimizations will be
-        // performed is ultimately up to the backend, but at least x86 does
-        // perform them.
-        let s0 = self.select(less_or_nan, int_min, fptosui_result);
-        let s1 = self.select(greater, int_max, s0);
-
-        // Step 3: NaN replacement.
-        // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN.
-        // Therefore we only need to execute this step for signed integer types.
-        if signed {
-            // LLVM has no isNaN predicate, so we use (x == x) instead
-            let cmp = self.fcmp(RealPredicate::RealOEQ, x, x);
-            self.select(cmp, s1, zero)
-        } else {
-            s1
-        }
+        if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) }
     }
 
     fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs
index 8a91d4735..fdc7a30e8 100644
--- a/compiler/rustc_codegen_ssa/src/traits/consts.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs
@@ -29,7 +29,6 @@ pub trait ConstMethods<'tcx>: BackendTypes {
     fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value;
 
     fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value;
-    fn zst_to_backend(&self, llty: Self::Type) -> Self::Value;
     fn from_const_alloc(
         &self,
         layout: TyAndLayout<'tcx>,