Merging upstream version 1.75.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

21 files changed, 719 insertions, 281 deletions

diff --git a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs
new file mode 100644
index 000000000..16bb7b12b
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs
@@ -0,0 +1,295 @@
+//! This pass is only used for UNIT TESTS related to incremental
+//! compilation. It tests whether a particular `.o` file will be re-used
+//! from a previous compilation or whether it must be regenerated.
+//!
+//! The user adds annotations to the crate of the following form:
+//!
+//! ```
+//! # #![feature(rustc_attrs)]
+//! # #![allow(internal_features)]
+//! #![rustc_partition_reused(module="spike", cfg="rpass2")]
+//! #![rustc_partition_codegened(module="spike-x", cfg="rpass2")]
+//! ```
+//!
+//! The first indicates (in the cfg `rpass2`) that `spike.o` will be
+//! reused, the second that `spike-x.o` will be recreated. If these
+//! annotations are inaccurate, errors are reported.
+//!
+//! The reason that we use `cfg=...` and not `#[cfg_attr]` is so that
+//! the HIR doesn't change as a result of the annotations, which might
+//! perturb the reuse results.
+//!
+//! `#![rustc_expected_cgu_reuse(module="spike", cfg="rpass2", kind="post-lto")]`
+//! allows for doing a more fine-grained check to see if pre- or post-lto data
+//! was re-used.
+
+use crate::errors;
+use rustc_ast as ast;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::unord::UnordSet;
+use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg};
+use rustc_hir::def_id::LOCAL_CRATE;
+use rustc_middle::mir::mono::CodegenUnitNameBuilder;
+use rustc_middle::ty::TyCtxt;
+use rustc_session::Session;
+use rustc_span::symbol::sym;
+use rustc_span::{Span, Symbol};
+use std::borrow::Cow;
+use std::fmt;
+use thin_vec::ThinVec;
+
+#[allow(missing_docs)]
+pub fn assert_module_sources(tcx: TyCtxt<'_>, set_reuse: &dyn Fn(&mut CguReuseTracker)) {
+    tcx.dep_graph.with_ignore(|| {
+        if tcx.sess.opts.incremental.is_none() {
+            return;
+        }
+
+        let available_cgus =
+            tcx.collect_and_partition_mono_items(()).1.iter().map(|cgu| cgu.name()).collect();
+
+        let mut ams = AssertModuleSource {
+            tcx,
+            available_cgus,
+            cgu_reuse_tracker: if tcx.sess.opts.unstable_opts.query_dep_graph {
+                CguReuseTracker::new()
+            } else {
+                CguReuseTracker::new_disabled()
+            },
+        };
+
+        for attr in tcx.hir().attrs(rustc_hir::CRATE_HIR_ID) {
+            ams.check_attr(attr);
+        }
+
+        set_reuse(&mut ams.cgu_reuse_tracker);
+
+        ams.cgu_reuse_tracker.check_expected_reuse(tcx.sess);
+    });
+}
+
+struct AssertModuleSource<'tcx> {
+    tcx: TyCtxt<'tcx>,
+    available_cgus: UnordSet<Symbol>,
+    cgu_reuse_tracker: CguReuseTracker,
+}
+
+impl<'tcx> AssertModuleSource<'tcx> {
+    fn check_attr(&mut self, attr: &ast::Attribute) {
+        let (expected_reuse, comp_kind) = if attr.has_name(sym::rustc_partition_reused) {
+            (CguReuse::PreLto, ComparisonKind::AtLeast)
+        } else if attr.has_name(sym::rustc_partition_codegened) {
+            (CguReuse::No, ComparisonKind::Exact)
+        } else if attr.has_name(sym::rustc_expected_cgu_reuse) {
+            match self.field(attr, sym::kind) {
+                sym::no => (CguReuse::No, ComparisonKind::Exact),
+                sym::pre_dash_lto => (CguReuse::PreLto, ComparisonKind::Exact),
+                sym::post_dash_lto => (CguReuse::PostLto, ComparisonKind::Exact),
+                sym::any => (CguReuse::PreLto, ComparisonKind::AtLeast),
+                other => {
+                    self.tcx
+                        .sess
+                        .emit_fatal(errors::UnknownReuseKind { span: attr.span, kind: other });
+                }
+            }
+        } else {
+            return;
+        };
+
+        if !self.tcx.sess.opts.unstable_opts.query_dep_graph {
+            self.tcx.sess.emit_fatal(errors::MissingQueryDepGraph { span: attr.span });
+        }
+
+        if !self.check_config(attr) {
+            debug!("check_attr: config does not match, ignoring attr");
+            return;
+        }
+
+        let user_path = self.field(attr, sym::module).to_string();
+        let crate_name = self.tcx.crate_name(LOCAL_CRATE).to_string();
+
+        if !user_path.starts_with(&crate_name) {
+            self.tcx.sess.emit_fatal(errors::MalformedCguName {
+                span: attr.span,
+                user_path,
+                crate_name,
+            });
+        }
+
+        // Split of the "special suffix" if there is one.
+        let (user_path, cgu_special_suffix) = if let Some(index) = user_path.rfind('.') {
+            (&user_path[..index], Some(&user_path[index + 1..]))
+        } else {
+            (&user_path[..], None)
+        };
+
+        let mut iter = user_path.split('-');
+
+        // Remove the crate name
+        assert_eq!(iter.next().unwrap(), crate_name);
+
+        let cgu_path_components = iter.collect::<Vec<_>>();
+
+        let cgu_name_builder = &mut CodegenUnitNameBuilder::new(self.tcx);
+        let cgu_name =
+            cgu_name_builder.build_cgu_name(LOCAL_CRATE, cgu_path_components, cgu_special_suffix);
+
+        debug!("mapping '{}' to cgu name '{}'", self.field(attr, sym::module), cgu_name);
+
+        if !self.available_cgus.contains(&cgu_name) {
+            let cgu_names: Vec<&str> =
+                self.available_cgus.items().map(|cgu| cgu.as_str()).into_sorted_stable_ord();
+            self.tcx.sess.emit_err(errors::NoModuleNamed {
+                span: attr.span,
+                user_path,
+                cgu_name,
+                cgu_names: cgu_names.join(", "),
+            });
+        }
+
+        self.cgu_reuse_tracker.set_expectation(
+            cgu_name,
+            &user_path,
+            attr.span,
+            expected_reuse,
+            comp_kind,
+        );
+    }
+
+    fn field(&self, attr: &ast::Attribute, name: Symbol) -> Symbol {
+        for item in attr.meta_item_list().unwrap_or_else(ThinVec::new) {
+            if item.has_name(name) {
+                if let Some(value) = item.value_str() {
+                    return value;
+                } else {
+                    self.tcx.sess.emit_fatal(errors::FieldAssociatedValueExpected {
+                        span: item.span(),
+                        name,
+                    });
+                }
+            }
+        }
+
+        self.tcx.sess.emit_fatal(errors::NoField { span: attr.span, name });
+    }
+
+    /// Scan for a `cfg="foo"` attribute and check whether we have a
+    /// cfg flag called `foo`.
+    fn check_config(&self, attr: &ast::Attribute) -> bool {
+        let config = &self.tcx.sess.parse_sess.config;
+        let value = self.field(attr, sym::cfg);
+        debug!("check_config(config={:?}, value={:?})", config, value);
+        if config.iter().any(|&(name, _)| name == value) {
+            debug!("check_config: matched");
+            return true;
+        }
+        debug!("check_config: no match found");
+        false
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
+pub enum CguReuse {
+    No,
+    PreLto,
+    PostLto,
+}
+
+impl fmt::Display for CguReuse {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match *self {
+            CguReuse::No => write!(f, "No"),
+            CguReuse::PreLto => write!(f, "PreLto "),
+            CguReuse::PostLto => write!(f, "PostLto "),
+        }
+    }
+}
+
+impl IntoDiagnosticArg for CguReuse {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        DiagnosticArgValue::Str(Cow::Owned(self.to_string()))
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub enum ComparisonKind {
+    Exact,
+    AtLeast,
+}
+
+struct TrackerData {
+    actual_reuse: FxHashMap<String, CguReuse>,
+    expected_reuse: FxHashMap<String, (String, Span, CguReuse, ComparisonKind)>,
+}
+
+pub struct CguReuseTracker {
+    data: Option<TrackerData>,
+}
+
+impl CguReuseTracker {
+    fn new() -> CguReuseTracker {
+        let data =
+            TrackerData { actual_reuse: Default::default(), expected_reuse: Default::default() };
+
+        CguReuseTracker { data: Some(data) }
+    }
+
+    fn new_disabled() -> CguReuseTracker {
+        CguReuseTracker { data: None }
+    }
+
+    pub fn set_actual_reuse(&mut self, cgu_name: &str, kind: CguReuse) {
+        if let Some(data) = &mut self.data {
+            debug!("set_actual_reuse({cgu_name:?}, {kind:?})");
+
+            let prev_reuse = data.actual_reuse.insert(cgu_name.to_string(), kind);
+            assert!(prev_reuse.is_none());
+        }
+    }
+
+    fn set_expectation(
+        &mut self,
+        cgu_name: Symbol,
+        cgu_user_name: &str,
+        error_span: Span,
+        expected_reuse: CguReuse,
+        comparison_kind: ComparisonKind,
+    ) {
+        if let Some(data) = &mut self.data {
+            debug!("set_expectation({cgu_name:?}, {expected_reuse:?}, {comparison_kind:?})");
+
+            data.expected_reuse.insert(
+                cgu_name.to_string(),
+                (cgu_user_name.to_string(), error_span, expected_reuse, comparison_kind),
+            );
+        }
+    }
+
+    fn check_expected_reuse(&self, sess: &Session) {
+        if let Some(ref data) = self.data {
+            for (cgu_name, &(ref cgu_user_name, ref error_span, expected_reuse, comparison_kind)) in
+                &data.expected_reuse
+            {
+                if let Some(&actual_reuse) = data.actual_reuse.get(cgu_name) {
+                    let (error, at_least) = match comparison_kind {
+                        ComparisonKind::Exact => (expected_reuse != actual_reuse, false),
+                        ComparisonKind::AtLeast => (actual_reuse < expected_reuse, true),
+                    };
+
+                    if error {
+                        let at_least = if at_least { 1 } else { 0 };
+                        errors::IncorrectCguReuseType {
+                            span: *error_span,
+                            cgu_user_name,
+                            actual_reuse,
+                            expected_reuse,
+                            at_least,
+                        };
+                    }
+                } else {
+                    sess.emit_fatal(errors::CguNotRecorded { cgu_user_name, cgu_name });
+                }
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index c4a0f6291..dd9d277fb 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -22,7 +22,9 @@ use rustc_session::utils::NativeLibKind;
 /// need out of the shared crate context before we get rid of it.
 use rustc_session::{filesearch, Session};
 use rustc_span::symbol::Symbol;
-use rustc_target::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault};
+use rustc_target::spec::crt_objects::CrtObjects;
+use rustc_target::spec::LinkSelfContainedComponents;
+use rustc_target::spec::LinkSelfContainedDefault;
 use rustc_target::spec::{Cc, LinkOutputKind, LinkerFlavor, Lld, PanicStrategy};
 use rustc_target::spec::{RelocModel, RelroLevel, SanitizerSet, SplitDebuginfo};
 
@@ -368,17 +370,25 @@ fn link_rlib<'a>(
         let NativeLibKind::Static { bundle: None | Some(true), .. } = lib.kind else {
             continue;
         };
-        if flavor == RlibFlavor::Normal && let Some(filename) = lib.filename {
+        if flavor == RlibFlavor::Normal
+            && let Some(filename) = lib.filename
+        {
             let path = find_native_static_library(filename.as_str(), true, &lib_search_paths, sess);
-            let src = read(path).map_err(|e| sess.emit_fatal(errors::ReadFileError {message: e }))?;
+            let src =
+                read(path).map_err(|e| sess.emit_fatal(errors::ReadFileError { message: e }))?;
             let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
             let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
             packed_bundled_libs.push(wrapper_file);
         } else {
-            let path =
-                find_native_static_library(lib.name.as_str(), lib.verbatim, &lib_search_paths, sess);
+            let path = find_native_static_library(
+                lib.name.as_str(),
+                lib.verbatim,
+                &lib_search_paths,
+                sess,
+            );
             ab.add_archive(&path, Box::new(|_| false)).unwrap_or_else(|error| {
-                sess.emit_fatal(errors::AddNativeLibrary { library_path: path, error })});
+                sess.emit_fatal(errors::AddNativeLibrary { library_path: path, error })
+            });
         }
     }
 
@@ -720,6 +730,7 @@ fn link_natively<'a>(
 ) -> Result<(), ErrorGuaranteed> {
     info!("preparing {:?} to {:?}", crate_type, out_filename);
     let (linker_path, flavor) = linker_and_flavor(sess);
+    let self_contained_components = self_contained_components(sess, crate_type);
     let mut cmd = linker_with_args(
         &linker_path,
         flavor,
@@ -729,6 +740,7 @@ fn link_natively<'a>(
         tmpdir,
         out_filename,
         codegen_results,
+        self_contained_components,
     )?;
 
     linker::disable_localization(&mut cmd);
@@ -804,14 +816,14 @@ fn link_natively<'a>(
                 "Linker does not support -static-pie command line option. Retrying with -static instead."
             );
             // Mirror `add_(pre,post)_link_objects` to replace CRT objects.
-            let self_contained = self_contained(sess, crate_type);
+            let self_contained_crt_objects = self_contained_components.is_crt_objects_enabled();
             let opts = &sess.target;
-            let pre_objects = if self_contained {
+            let pre_objects = if self_contained_crt_objects {
                 &opts.pre_link_objects_self_contained
             } else {
                 &opts.pre_link_objects
             };
-            let post_objects = if self_contained {
+            let post_objects = if self_contained_crt_objects {
                 &opts.post_link_objects_self_contained
             } else {
                 &opts.post_link_objects
@@ -822,7 +834,9 @@ fn link_natively<'a>(
                     .iter()
                     .copied()
                     .flatten()
-                    .map(|obj| get_object_file_path(sess, obj, self_contained).into_os_string())
+                    .map(|obj| {
+                        get_object_file_path(sess, obj, self_contained_crt_objects).into_os_string()
+                    })
                     .collect::<Vec<_>>()
             };
             let pre_objects_static_pie = get_objects(pre_objects, LinkOutputKind::StaticPicExe);
@@ -1019,7 +1033,7 @@ fn link_natively<'a>(
         SplitDebuginfo::Packed => link_dwarf_object(sess, codegen_results, out_filename),
     }
 
-    let strip = strip_value(sess);
+    let strip = sess.opts.cg.strip;
 
     if sess.target.is_like_osx {
         match (strip, crate_type) {
@@ -1056,14 +1070,6 @@ fn link_natively<'a>(
     Ok(())
 }
 
-// Temporarily support both -Z strip and -C strip
-fn strip_value(sess: &Session) -> Strip {
-    match (sess.opts.unstable_opts.strip, sess.opts.cg.strip) {
-        (s, Strip::None) => s,
-        (_, s) => s,
-    }
-}
-
 fn strip_symbols_with_external_utility<'a>(
     sess: &'a Session,
     util: &str,
@@ -1702,26 +1708,43 @@ fn detect_self_contained_mingw(sess: &Session) -> bool {
 /// Various toolchain components used during linking are used from rustc distribution
 /// instead of being found somewhere on the host system.
 /// We only provide such support for a very limited number of targets.
-fn self_contained(sess: &Session, crate_type: CrateType) -> bool {
-    if let Some(self_contained) = sess.opts.cg.link_self_contained.explicitly_set {
-        if sess.target.link_self_contained == LinkSelfContainedDefault::False {
-            sess.emit_err(errors::UnsupportedLinkSelfContained);
-        }
-        return self_contained;
-    }
+fn self_contained_components(sess: &Session, crate_type: CrateType) -> LinkSelfContainedComponents {
+    // Turn the backwards compatible bool values for `self_contained` into fully inferred
+    // `LinkSelfContainedComponents`.
+    let self_contained =
+        if let Some(self_contained) = sess.opts.cg.link_self_contained.explicitly_set {
+            // Emit an error if the user requested self-contained mode on the CLI but the target
+            // explicitly refuses it.
+            if sess.target.link_self_contained.is_disabled() {
+                sess.emit_err(errors::UnsupportedLinkSelfContained);
+            }
+            self_contained
+        } else {
+            match sess.target.link_self_contained {
+                LinkSelfContainedDefault::False => false,
+                LinkSelfContainedDefault::True => true,
+
+                LinkSelfContainedDefault::WithComponents(components) => {
+                    // For target specs with explicitly enabled components, we can return them
+                    // directly.
+                    return components;
+                }
 
-    match sess.target.link_self_contained {
-        LinkSelfContainedDefault::False => false,
-        LinkSelfContainedDefault::True => true,
-        // FIXME: Find a better heuristic for "native musl toolchain is available",
-        // based on host and linker path, for example.
-        // (https://github.com/rust-lang/rust/pull/71769#issuecomment-626330237).
-        LinkSelfContainedDefault::Musl => sess.crt_static(Some(crate_type)),
-        LinkSelfContainedDefault::Mingw => {
-            sess.host == sess.target
-                && sess.target.vendor != "uwp"
-                && detect_self_contained_mingw(&sess)
-        }
+                // FIXME: Find a better heuristic for "native musl toolchain is available",
+                // based on host and linker path, for example.
+                // (https://github.com/rust-lang/rust/pull/71769#issuecomment-626330237).
+                LinkSelfContainedDefault::InferredForMusl => sess.crt_static(Some(crate_type)),
+                LinkSelfContainedDefault::InferredForMingw => {
+                    sess.host == sess.target
+                        && sess.target.vendor != "uwp"
+                        && detect_self_contained_mingw(&sess)
+                }
+            }
+        };
+    if self_contained {
+        LinkSelfContainedComponents::all()
+    } else {
+        LinkSelfContainedComponents::empty()
     }
 }
 
@@ -1881,37 +1904,14 @@ fn add_linked_symbol_object(
         return;
     };
 
-    // NOTE(nbdd0121): MSVC will hang if the input object file contains no sections,
-    // so add an empty section.
     if file.format() == object::BinaryFormat::Coff {
+        // NOTE(nbdd0121): MSVC will hang if the input object file contains no sections,
+        // so add an empty section.
         file.add_section(Vec::new(), ".text".into(), object::SectionKind::Text);
 
         // We handle the name decoration of COFF targets in `symbol_export.rs`, so disable the
         // default mangler in `object` crate.
         file.set_mangling(object::write::Mangling::None);
-
-        // Add feature flags to the object file. On MSVC this is optional but LLD will complain if
-        // not present.
-        let mut feature = 0;
-
-        if file.architecture() == object::Architecture::I386 {
-            // Indicate that all SEH handlers are registered in .sxdata section.
-            // We don't have generate any code, so we don't need .sxdata section but LLD still
-            // expects us to set this bit (see #96498).
-            // Reference: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
-            feature |= 1;
-        }
-
-        file.add_symbol(object::write::Symbol {
-            name: "@feat.00".into(),
-            value: feature,
-            size: 0,
-            kind: object::SymbolKind::Data,
-            scope: object::SymbolScope::Compilation,
-            weak: false,
-            section: object::write::SymbolSection::Absolute,
-            flags: object::SymbolFlags::None,
-        });
     }
 
     for (sym, kind) in symbols.iter() {
@@ -2045,13 +2045,14 @@ fn linker_with_args<'a>(
     tmpdir: &Path,
     out_filename: &Path,
     codegen_results: &CodegenResults,
+    self_contained_components: LinkSelfContainedComponents,
 ) -> Result<Command, ErrorGuaranteed> {
-    let self_contained = self_contained(sess, crate_type);
+    let self_contained_crt_objects = self_contained_components.is_crt_objects_enabled();
     let cmd = &mut *super::linker::get_linker(
         sess,
         path,
         flavor,
-        self_contained,
+        self_contained_components.are_any_components_enabled(),
         &codegen_results.crate_info.target_cpu,
     );
     let link_output_kind = link_output_kind(sess, crate_type);
@@ -2078,7 +2079,7 @@ fn linker_with_args<'a>(
     // ------------ Object code and libraries, order-dependent ------------
 
     // Pre-link CRT objects.
-    add_pre_link_objects(cmd, sess, flavor, link_output_kind, self_contained);
+    add_pre_link_objects(cmd, sess, flavor, link_output_kind, self_contained_crt_objects);
 
     add_linked_symbol_object(
         cmd,
@@ -2221,7 +2222,7 @@ fn linker_with_args<'a>(
         cmd,
         sess,
         link_output_kind,
-        self_contained,
+        self_contained_components,
         flavor,
         crate_type,
         codegen_results,
@@ -2237,7 +2238,7 @@ fn linker_with_args<'a>(
     // ------------ Object code and libraries, order-dependent ------------
 
     // Post-link CRT objects.
-    add_post_link_objects(cmd, sess, link_output_kind, self_contained);
+    add_post_link_objects(cmd, sess, link_output_kind, self_contained_crt_objects);
 
     // ------------ Late order-dependent options ------------
 
@@ -2254,7 +2255,7 @@ fn add_order_independent_options(
     cmd: &mut dyn Linker,
     sess: &Session,
     link_output_kind: LinkOutputKind,
-    self_contained: bool,
+    self_contained_components: LinkSelfContainedComponents,
     flavor: LinkerFlavor,
     crate_type: CrateType,
     codegen_results: &CodegenResults,
@@ -2262,7 +2263,7 @@ fn add_order_independent_options(
     tmpdir: &Path,
 ) {
     // Take care of the flavors and CLI options requesting the `lld` linker.
-    add_lld_args(cmd, sess, flavor);
+    add_lld_args(cmd, sess, flavor, self_contained_components);
 
     add_apple_sdk(cmd, sess, flavor);
 
@@ -2287,7 +2288,7 @@ fn add_order_independent_options(
     // Make the binary compatible with data execution prevention schemes.
     cmd.add_no_exec();
 
-    if self_contained {
+    if self_contained_components.is_crt_objects_enabled() {
         cmd.no_crt_objects();
     }
 
@@ -2318,7 +2319,7 @@ fn add_order_independent_options(
 
     cmd.linker_plugin_lto();
 
-    add_library_search_dirs(cmd, sess, self_contained);
+    add_library_search_dirs(cmd, sess, self_contained_components.are_any_components_enabled());
 
     cmd.output_filename(out_filename);
 
@@ -2361,7 +2362,7 @@ fn add_order_independent_options(
     );
 
     // Pass debuginfo, NatVis debugger visualizers and strip flags down to the linker.
-    cmd.debuginfo(strip_value(sess), &natvis_visualizers);
+    cmd.debuginfo(sess.opts.cg.strip, &natvis_visualizers);
 
     // We want to prevent the compiler from accidentally leaking in any system libraries,
     // so by default we tell linkers not to link to any default libraries.
@@ -2871,6 +2872,7 @@ fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
     }
 
     let sdk_name = match (arch.as_ref(), os.as_ref()) {
+        ("aarch64", "tvos") if llvm_target.ends_with("-simulator") => "appletvsimulator",
         ("aarch64", "tvos") => "appletvos",
         ("x86_64", "tvos") => "appletvsimulator",
         ("arm", "ios") => "iphoneos",
@@ -2964,31 +2966,54 @@ fn get_apple_sdk_root(sdk_name: &str) -> Result<String, errors::AppleSdkRootErro
     }
 }
 
-/// When using the linker flavors opting in to `lld`, or the unstable `-Zgcc-ld=lld` flag, add the
-/// necessary paths and arguments to invoke it:
+/// When using the linker flavors opting in to `lld`, add the necessary paths and arguments to
+/// invoke it:
 /// - when the self-contained linker flag is active: the build of `lld` distributed with rustc,
 /// - or any `lld` available to `cc`.
-fn add_lld_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
-    let unstable_use_lld = sess.opts.unstable_opts.gcc_ld.is_some();
-    debug!("add_lld_args requested, flavor: '{flavor:?}', `-Zgcc-ld=lld`: {unstable_use_lld}");
-
-    // Sanity check: using the old unstable `-Zgcc-ld=lld` option requires a `cc`-using flavor.
-    let flavor_uses_cc = flavor.uses_cc();
-    if unstable_use_lld && !flavor_uses_cc {
-        sess.emit_fatal(errors::OptionGccOnly);
-    }
+fn add_lld_args(
+    cmd: &mut dyn Linker,
+    sess: &Session,
+    flavor: LinkerFlavor,
+    self_contained_components: LinkSelfContainedComponents,
+) {
+    debug!(
+        "add_lld_args requested, flavor: '{:?}', target self-contained components: {:?}",
+        flavor, self_contained_components,
+    );
 
     // If the flavor doesn't use a C/C++ compiler to invoke the linker, or doesn't opt in to `lld`,
     // we don't need to do anything.
-    let use_lld = flavor.uses_lld() || unstable_use_lld;
-    if !flavor_uses_cc || !use_lld {
+    if !(flavor.uses_cc() && flavor.uses_lld()) {
         return;
     }
 
     // 1. Implement the "self-contained" part of this feature by adding rustc distribution
-    //    directories to the tool's search path.
-    let self_contained_linker = sess.opts.cg.link_self_contained.linker() || unstable_use_lld;
-    if self_contained_linker {
+    // directories to the tool's search path, depending on a mix between what users can specify on
+    // the CLI, and what the target spec enables (as it can't disable components):
+    // - if the self-contained linker is enabled on the CLI or by the target spec,
+    // - and if the self-contained linker is not disabled on the CLI.
+    let self_contained_cli = sess.opts.cg.link_self_contained.is_linker_enabled();
+    let self_contained_target = self_contained_components.is_linker_enabled();
+
+    // FIXME: in the future, codegen backends may need to have more control over this process: they
+    // don't always support all the features the linker expects here, and vice versa. For example,
+    // at the time of writing this, lld expects a newer style of aarch64 TLS relocations that
+    // cranelift doesn't implement yet. That in turn can impact whether linking would succeed on
+    // such a target when using the `cg_clif` backend and lld.
+    //
+    // Until interactions between backends and linker features are expressible, we limit target
+    // specs to opt-in to lld only when we're on the llvm backend, where it's expected to work and
+    // tested on CI. As usual, the CLI still has precedence over this, so that users and developers
+    // can still override this default when needed (e.g. for tests).
+    let uses_llvm_backend =
+        matches!(sess.opts.unstable_opts.codegen_backend.as_deref(), None | Some("llvm"));
+    if !uses_llvm_backend && !self_contained_cli && sess.opts.cg.linker_flavor.is_none() {
+        // We bail if we're not using llvm and lld was not explicitly requested on the CLI.
+        return;
+    }
+
+    let self_contained_linker = self_contained_cli || self_contained_target;
+    if self_contained_linker && !sess.opts.cg.link_self_contained.is_linker_disabled() {
         for path in sess.get_tools_search_paths(false) {
             cmd.arg({
                 let mut arg = OsString::from("-B");
@@ -2999,7 +3024,7 @@ fn add_lld_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
     }
 
     // 2. Implement the "linker flavor" part of this feature by asking `cc` to use some kind of
-    //    `lld` as the linker.
+    // `lld` as the linker.
     cmd.arg("-fuse-ld=lld");
 
     if !flavor.is_gnu() {
@@ -3012,13 +3037,13 @@ fn add_lld_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
         // shown in issue #101653 and the discussion in PR #101792.
         //
         // It could be required in some cases of cross-compiling with
-        // `-Zgcc-ld=lld`, but this is generally unspecified, and we don't know
+        // LLD, but this is generally unspecified, and we don't know
         // which specific versions of clang, macOS SDK, host and target OS
         // combinations impact us here.
         //
         // So we do a simple first-approximation until we know more of what the
         // Apple targets require (and which would be handled prior to hitting this
-        // `-Zgcc-ld=lld` codepath anyway), but the expectation is that until then
+        // LLD codepath anyway), but the expectation is that until then
         // this should be manually passed if needed. We specify the target when
         // targeting a different linker flavor on macOS, and that's also always
         // the case when targeting WASM.
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index 11afe0fbc..09434513e 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -626,6 +626,15 @@ impl<'a> Linker for GccLinker<'a> {
                 self.linker_arg("--strip-all");
             }
         }
+        match self.sess.opts.unstable_opts.debuginfo_compression {
+            config::DebugInfoCompression::None => {}
+            config::DebugInfoCompression::Zlib => {
+                self.linker_arg("--compress-debug-sections=zlib");
+            }
+            config::DebugInfoCompression::Zstd => {
+                self.linker_arg("--compress-debug-sections=zstd");
+            }
+        }
     }
 
     fn no_crt_objects(&mut self) {
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index c6f4bd35e..cb60ed729 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -228,6 +228,35 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
     if sess.target.is_like_osx {
         file.set_macho_build_version(macho_object_build_version_for_target(&sess.target))
     }
+    if binary_format == BinaryFormat::Coff {
+        // Disable the default mangler to avoid mangling the special "@feat.00" symbol name.
+        let original_mangling = file.mangling();
+        file.set_mangling(object::write::Mangling::None);
+
+        let mut feature = 0;
+
+        if file.architecture() == object::Architecture::I386 {
+            // When linking with /SAFESEH on x86, lld requires that all linker inputs be marked as
+            // safe exception handling compatible. Metadata files masquerade as regular COFF
+            // objects and are treated as linker inputs, despite containing no actual code. Thus,
+            // they still need to be marked as safe exception handling compatible. See #96498.
+            // Reference: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
+            feature |= 1;
+        }
+
+        file.add_symbol(object::write::Symbol {
+            name: "@feat.00".into(),
+            value: feature,
+            size: 0,
+            kind: object::SymbolKind::Data,
+            scope: object::SymbolScope::Compilation,
+            weak: false,
+            section: object::write::SymbolSection::Absolute,
+            flags: object::SymbolFlags::None,
+        });
+
+        file.set_mangling(original_mangling);
+    }
     let e_flags = match architecture {
         Architecture::Mips => {
             let arch = match sess.target.options.cpu.as_ref() {
diff --git a/compiler/rustc_codegen_ssa/src/back/rpath.rs b/compiler/rustc_codegen_ssa/src/back/rpath.rs
index ebf04e7a3..603462286 100644
--- a/compiler/rustc_codegen_ssa/src/back/rpath.rs
+++ b/compiler/rustc_codegen_ssa/src/back/rpath.rs
@@ -1,8 +1,7 @@
 use pathdiff::diff_paths;
 use rustc_data_structures::fx::FxHashSet;
-use std::env;
+use rustc_fs_util::try_canonicalize;
 use std::ffi::OsString;
-use std::fs;
 use std::path::{Path, PathBuf};
 
 pub struct RPathConfig<'a> {
@@ -82,12 +81,11 @@ fn get_rpath_relative_to_output(config: &mut RPathConfig<'_>, lib: &Path) -> OsS
     // Mac doesn't appear to support $ORIGIN
     let prefix = if config.is_like_osx { "@loader_path" } else { "$ORIGIN" };
 
-    let cwd = env::current_dir().unwrap();
-    let mut lib = fs::canonicalize(&cwd.join(lib)).unwrap_or_else(|_| cwd.join(lib));
-    lib.pop(); // strip filename
-    let mut output = cwd.join(&config.out_filename);
-    output.pop(); // strip filename
-    let output = fs::canonicalize(&output).unwrap_or(output);
+    // Strip filenames
+    let lib = lib.parent().unwrap();
+    let output = config.out_filename.parent().unwrap();
+    let lib = try_canonicalize(lib).unwrap();
+    let output = try_canonicalize(output).unwrap();
     let relative = path_relative_from(&lib, &output)
         .unwrap_or_else(|| panic!("couldn't create relative path from {output:?} to {lib:?}"));
 
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index f192747c8..3d6a21243 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -26,7 +26,6 @@ use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::middle::exported_symbols::SymbolExportInfo;
 use rustc_middle::ty::TyCtxt;
-use rustc_session::cgu_reuse_tracker::CguReuseTracker;
 use rustc_session::config::{self, CrateType, Lto, OutFileName, OutputFilenames, OutputType};
 use rustc_session::config::{Passes, SwitchWithOptPath};
 use rustc_session::Session;
@@ -366,8 +365,6 @@ pub struct CodegenContext<B: WriteBackendMethods> {
     /// The incremental compilation session directory, or None if we are not
     /// compiling incrementally
     pub incr_comp_session_dir: Option<PathBuf>,
-    /// Used to update CGU re-use information during the thinlto phase.
-    pub cgu_reuse_tracker: CguReuseTracker,
     /// Channel back to the main control thread to send messages to
     pub coordinator_send: Sender<Box<dyn Any + Send>>,
 }
@@ -1119,7 +1116,6 @@ fn start_executing_work<B: ExtraBackendMethods>(
         remark: sess.opts.cg.remark.clone(),
         remark_dir,
         incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
-        cgu_reuse_tracker: sess.cgu_reuse_tracker.clone(),
         coordinator_send,
         expanded_args: tcx.sess.expanded_args.clone(),
         diag_emitter: shared_emitter.clone(),
@@ -1969,8 +1965,6 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> {
             }
         });
 
-        sess.cgu_reuse_tracker.check_expected_reuse(sess);
-
         sess.abort_if_errors();
 
         let work_products =
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 1e4ea73a1..198e56963 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -1,3 +1,4 @@
+use crate::assert_module_sources::CguReuse;
 use crate::back::link::are_upstream_rust_objects_already_included;
 use crate::back::metadata::create_compressed_metadata_file;
 use crate::back::write::{
@@ -31,7 +32,6 @@ use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem};
 use rustc_middle::query::Providers;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
-use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{self, CrateType, EntryFnType, OutputType};
 use rustc_session::Session;
 use rustc_span::symbol::sym;
@@ -683,6 +683,13 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
         codegen_units.iter().map(|cgu| determine_cgu_reuse(tcx, &cgu)).collect::<Vec<_>>()
     });
 
+    crate::assert_module_sources::assert_module_sources(tcx, &|cgu_reuse_tracker| {
+        for (i, cgu) in codegen_units.iter().enumerate() {
+            let cgu_reuse = cgu_reuse[i];
+            cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse);
+        }
+    });
+
     let mut total_codegen_time = Duration::new(0, 0);
     let start_rss = tcx.sess.opts.unstable_opts.time_passes.then(|| get_resident_set_size());
 
@@ -727,7 +734,6 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
         ongoing_codegen.check_for_errors(tcx.sess);
 
         let cgu_reuse = cgu_reuse[i];
-        tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse);
 
         match cgu_reuse {
             CguReuse::No => {
@@ -994,7 +1000,7 @@ pub fn provide(providers: &mut Providers) {
     };
 }
 
-fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse {
+pub fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse {
     if !tcx.dep_graph.is_fully_enabled() {
         return CguReuse::No;
     }
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 59efe4cd3..2e0840f2d 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -238,8 +238,13 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                     && let Some(fn_sig) = fn_sig()
                     && fn_sig.skip_binder().abi() != abi::Abi::Rust
                 {
-                    struct_span_err!(tcx.sess, attr.span, E0737, "`#[track_caller]` requires Rust ABI")
-                        .emit();
+                    struct_span_err!(
+                        tcx.sess,
+                        attr.span,
+                        E0737,
+                        "`#[track_caller]` requires Rust ABI"
+                    )
+                    .emit();
                 }
                 if is_closure
                     && !tcx.features().closure_track_caller
@@ -435,17 +440,18 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                     && let [item] = items.as_slice()
                     && let Some((sym::align, literal)) = item.name_value_literal()
                 {
-                    rustc_attr::parse_alignment(&literal.kind).map_err(|msg| {
-                        struct_span_err!(
-                            tcx.sess.diagnostic(),
-                            attr.span,
-                            E0589,
-                            "invalid `repr(align)` attribute: {}",
-                            msg
-                        )
-                        .emit();
-                    })
-                    .ok()
+                    rustc_attr::parse_alignment(&literal.kind)
+                        .map_err(|msg| {
+                            struct_span_err!(
+                                tcx.sess.diagnostic(),
+                                attr.span,
+                                E0589,
+                                "invalid `repr(align)` attribute: {}",
+                                msg
+                            )
+                            .emit();
+                        })
+                        .ok()
                 } else {
                     None
                 };
@@ -626,10 +632,7 @@ fn should_inherit_track_caller(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
         && let ty::AssocItemContainer::ImplContainer = impl_item.container
         && let Some(trait_item) = impl_item.trait_item_def_id
     {
-        return tcx
-            .codegen_fn_attrs(trait_item)
-            .flags
-            .intersects(CodegenFnAttrFlags::TRACK_CALLER);
+        return tcx.codegen_fn_attrs(trait_item).flags.intersects(CodegenFnAttrFlags::TRACK_CALLER);
     }
 
     false
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 989df448a..1a85eb8dd 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -15,7 +15,7 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher};
 use rustc_hir::def_id::DefId;
 use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathData};
-use rustc_hir::{AsyncGeneratorKind, GeneratorKind, Mutability};
+use rustc_hir::{CoroutineKind, CoroutineSource, Mutability};
 use rustc_middle::ty::layout::{IntegerExt, TyAndLayout};
 use rustc_middle::ty::{self, ExistentialProjection, ParamEnv, Ty, TyCtxt};
 use rustc_middle::ty::{GenericArgKind, GenericArgsRef};
@@ -398,23 +398,23 @@ fn push_debuginfo_type_name<'tcx>(
             // processing
             visited.remove(&t);
         }
-        ty::Closure(def_id, args) | ty::Generator(def_id, args, ..) => {
-            // Name will be "{closure_env#0}<T1, T2, ...>", "{generator_env#0}<T1, T2, ...>", or
+        ty::Closure(def_id, args) | ty::Coroutine(def_id, args, ..) => {
+            // Name will be "{closure_env#0}<T1, T2, ...>", "{coroutine_env#0}<T1, T2, ...>", or
             // "{async_fn_env#0}<T1, T2, ...>", etc.
             // In the case of cpp-like debuginfo, the name additionally gets wrapped inside of
             // an artificial `enum2$<>` type, as defined in msvc_enum_fallback().
-            if cpp_like_debuginfo && t.is_generator() {
+            if cpp_like_debuginfo && t.is_coroutine() {
                 let ty_and_layout = tcx.layout_of(ParamEnv::reveal_all().and(t)).unwrap();
                 msvc_enum_fallback(
                     ty_and_layout,
                     &|output, visited| {
-                        push_closure_or_generator_name(tcx, def_id, args, true, output, visited);
+                        push_closure_or_coroutine_name(tcx, def_id, args, true, output, visited);
                     },
                     output,
                     visited,
                 );
             } else {
-                push_closure_or_generator_name(tcx, def_id, args, qualified, output, visited);
+                push_closure_or_coroutine_name(tcx, def_id, args, qualified, output, visited);
             }
         }
         // Type parameters from polymorphized functions.
@@ -426,7 +426,7 @@ fn push_debuginfo_type_name<'tcx>(
         | ty::Placeholder(..)
         | ty::Alias(..)
         | ty::Bound(..)
-        | ty::GeneratorWitness(..) => {
+        | ty::CoroutineWitness(..) => {
             bug!(
                 "debuginfo: Trying to create type name for \
                   unexpected type: {:?}",
@@ -558,12 +558,15 @@ pub fn push_item_name(tcx: TyCtxt<'_>, def_id: DefId, qualified: bool, output: &
     push_unqualified_item_name(tcx, def_id, def_key.disambiguated_data, output);
 }
 
-fn generator_kind_label(generator_kind: Option<GeneratorKind>) -> &'static str {
-    match generator_kind {
-        Some(GeneratorKind::Async(AsyncGeneratorKind::Block)) => "async_block",
-        Some(GeneratorKind::Async(AsyncGeneratorKind::Closure)) => "async_closure",
-        Some(GeneratorKind::Async(AsyncGeneratorKind::Fn)) => "async_fn",
-        Some(GeneratorKind::Gen) => "generator",
+fn coroutine_kind_label(coroutine_kind: Option<CoroutineKind>) -> &'static str {
+    match coroutine_kind {
+        Some(CoroutineKind::Gen(CoroutineSource::Block)) => "gen_block",
+        Some(CoroutineKind::Gen(CoroutineSource::Closure)) => "gen_closure",
+        Some(CoroutineKind::Gen(CoroutineSource::Fn)) => "gen_fn",
+        Some(CoroutineKind::Async(CoroutineSource::Block)) => "async_block",
+        Some(CoroutineKind::Async(CoroutineSource::Closure)) => "async_closure",
+        Some(CoroutineKind::Async(CoroutineSource::Fn)) => "async_fn",
+        Some(CoroutineKind::Coroutine) => "coroutine",
         None => "closure",
     }
 }
@@ -592,7 +595,7 @@ fn push_unqualified_item_name(
             output.push_str(tcx.crate_name(def_id.krate).as_str());
         }
         DefPathData::ClosureExpr => {
-            let label = generator_kind_label(tcx.generator_kind(def_id));
+            let label = coroutine_kind_label(tcx.coroutine_kind(def_id));
 
             push_disambiguated_special_name(
                 label,
@@ -707,7 +710,7 @@ pub fn push_generic_params<'tcx>(
     push_generic_params_internal(tcx, args, def_id, output, &mut visited);
 }
 
-fn push_closure_or_generator_name<'tcx>(
+fn push_closure_or_coroutine_name<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: DefId,
     args: GenericArgsRef<'tcx>,
@@ -715,10 +718,10 @@ fn push_closure_or_generator_name<'tcx>(
     output: &mut String,
     visited: &mut FxHashSet<Ty<'tcx>>,
 ) {
-    // Name will be "{closure_env#0}<T1, T2, ...>", "{generator_env#0}<T1, T2, ...>", or
+    // Name will be "{closure_env#0}<T1, T2, ...>", "{coroutine_env#0}<T1, T2, ...>", or
     // "{async_fn_env#0}<T1, T2, ...>", etc.
     let def_key = tcx.def_key(def_id);
-    let generator_kind = tcx.generator_kind(def_id);
+    let coroutine_kind = tcx.coroutine_kind(def_id);
 
     if qualified {
         let parent_def_id = DefId { index: def_key.parent.unwrap(), ..def_id };
@@ -727,7 +730,7 @@ fn push_closure_or_generator_name<'tcx>(
     }
 
     let mut label = String::with_capacity(20);
-    write!(&mut label, "{}_env", generator_kind_label(generator_kind)).unwrap();
+    write!(&mut label, "{}_env", coroutine_kind_label(coroutine_kind)).unwrap();
 
     push_disambiguated_special_name(
         &label,
@@ -736,7 +739,7 @@ fn push_closure_or_generator_name<'tcx>(
         output,
     );
 
-    // We also need to add the generic arguments of the async fn/generator or
+    // We also need to add the generic arguments of the async fn/coroutine or
     // the enclosing function (for closures or async blocks), so that we end
     // up with a unique name for every instantiation.
 
@@ -745,7 +748,7 @@ fn push_closure_or_generator_name<'tcx>(
     let generics = tcx.generics_of(enclosing_fn_def_id);
 
     // Truncate the args to the length of the above generics. This will cut off
-    // anything closure- or generator-specific.
+    // anything closure- or coroutine-specific.
     let args = args.truncate_to(tcx, generics);
     push_generic_params_internal(tcx, args, enclosing_fn_def_id, output, visited);
 }
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index 14311ec08..ed6ac9f9c 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -1,5 +1,6 @@
 //! Errors emitted by codegen_ssa
 
+use crate::assert_module_sources::CguReuse;
 use crate::back::command::Command;
 use crate::fluent_generated as fluent;
 use rustc_errors::{
@@ -17,6 +18,74 @@ use std::path::{Path, PathBuf};
 use std::process::ExitStatus;
 
 #[derive(Diagnostic)]
+#[diag(codegen_ssa_incorrect_cgu_reuse_type)]
+pub struct IncorrectCguReuseType<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub cgu_user_name: &'a str,
+    pub actual_reuse: CguReuse,
+    pub expected_reuse: CguReuse,
+    pub at_least: u8,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_cgu_not_recorded)]
+pub struct CguNotRecorded<'a> {
+    pub cgu_user_name: &'a str,
+    pub cgu_name: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_unknown_reuse_kind)]
+pub struct UnknownReuseKind {
+    #[primary_span]
+    pub span: Span,
+    pub kind: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_missing_query_depgraph)]
+pub struct MissingQueryDepGraph {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_malformed_cgu_name)]
+pub struct MalformedCguName {
+    #[primary_span]
+    pub span: Span,
+    pub user_path: String,
+    pub crate_name: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_no_module_named)]
+pub struct NoModuleNamed<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub user_path: &'a str,
+    pub cgu_name: Symbol,
+    pub cgu_names: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_field_associated_value_expected)]
+pub struct FieldAssociatedValueExpected {
+    #[primary_span]
+    pub span: Span,
+    pub name: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_no_field)]
+pub struct NoField {
+    #[primary_span]
+    pub span: Span,
+    pub name: Symbol,
+}
+
+#[derive(Diagnostic)]
 #[diag(codegen_ssa_lib_def_write_failure)]
 pub struct LibDefWriteFailure {
     pub error: Error,
@@ -490,10 +559,6 @@ pub struct RlibArchiveBuildFailure {
 }
 
 #[derive(Diagnostic)]
-#[diag(codegen_ssa_option_gcc_only)]
-pub struct OptionGccOnly;
-
-#[derive(Diagnostic)]
 pub enum ExtractBundledLibsError<'a> {
     #[diag(codegen_ssa_extract_bundled_libs_open_file)]
     OpenFile { rlib: &'a Path, error: Box<dyn std::error::Error> },
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index f6186a290..156c2904f 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -1,4 +1,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![cfg_attr(not(bootstrap), allow(internal_features))]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]
@@ -43,6 +46,7 @@ use std::collections::BTreeSet;
 use std::io;
 use std::path::{Path, PathBuf};
 
+pub mod assert_module_sources;
 pub mod back;
 pub mod base;
 pub mod codegen_attrs;
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index d9419dbc9..2285e7f4e 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -8,7 +8,7 @@ use rustc_index::bit_set::BitSet;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::mir::traversal;
 use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor};
-use rustc_middle::mir::{self, Location, TerminatorKind};
+use rustc_middle::mir::{self, DefLocation, Location, TerminatorKind};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 
 pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
@@ -67,21 +67,6 @@ enum LocalKind {
     SSA(DefLocation),
 }
 
-#[derive(Copy, Clone, PartialEq, Eq)]
-enum DefLocation {
-    Argument,
-    Body(Location),
-}
-
-impl DefLocation {
-    fn dominates(self, location: Location, dominators: &Dominators<mir::BasicBlock>) -> bool {
-        match self {
-            DefLocation::Argument => true,
-            DefLocation::Body(def) => def.successor_within_block().dominates(location, dominators),
-        }
-    }
-}
-
 struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
     fx: &'mir FunctionCx<'a, 'tcx, Bx>,
     dominators: &'mir Dominators<mir::BasicBlock>,
@@ -287,7 +272,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
                 | TerminatorKind::UnwindResume
                 | TerminatorKind::UnwindTerminate(_)
                 | TerminatorKind::Return
-                | TerminatorKind::GeneratorDrop
+                | TerminatorKind::CoroutineDrop
                 | TerminatorKind::Unreachable
                 | TerminatorKind::SwitchInt { .. }
                 | TerminatorKind::Yield { .. }
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index bd0707edf..3d2d8f8b5 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -17,8 +17,7 @@ use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_session::config::OptLevel;
-use rustc_span::source_map::Span;
-use rustc_span::{sym, Symbol};
+use rustc_span::{sym, Span, Symbol};
 use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode, Reg};
 use rustc_target::abi::{self, HasDataLayout, WrappingRange};
 use rustc_target::spec::abi::Abi;
@@ -213,7 +212,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
                 self.funclet(fx),
             );
             if fx.mir[self.bb].is_cleanup {
-                bx.do_not_inline(invokeret);
+                bx.apply_attrs_to_cleanup_callsite(invokeret);
             }
 
             if let Some((ret_dest, target)) = destination {
@@ -228,11 +227,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         } else {
             let llret = bx.call(fn_ty, fn_attrs, Some(&fn_abi), fn_ptr, &llargs, self.funclet(fx));
             if fx.mir[self.bb].is_cleanup {
-                // Cleanup is always the cold path. Don't inline
-                // drop glue. Also, when there is a deeply-nested
-                // struct, there are "symmetry" issues that cause
-                // exponential inlining - see issue #41696.
-                bx.do_not_inline(llret);
+                bx.apply_attrs_to_cleanup_callsite(llret);
             }
 
             if let Some((ret_dest, target)) = destination {
@@ -1269,8 +1264,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 fn_span,
                 mergeable_succ(),
             ),
-            mir::TerminatorKind::GeneratorDrop | mir::TerminatorKind::Yield { .. } => {
-                bug!("generator ops in codegen")
+            mir::TerminatorKind::CoroutineDrop | mir::TerminatorKind::Yield { .. } => {
+                bug!("coroutine ops in codegen")
             }
             mir::TerminatorKind::FalseEdge { .. } | mir::TerminatorKind::FalseUnwind { .. } => {
                 bug!("borrowck false edges in codegen")
@@ -1453,46 +1448,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     fn get_caller_location(
         &mut self,
         bx: &mut Bx,
-        mut source_info: mir::SourceInfo,
+        source_info: mir::SourceInfo,
     ) -> OperandRef<'tcx, Bx::Value> {
-        let tcx = bx.tcx();
-
-        let mut span_to_caller_location = |span: Span| {
-            let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
-            let caller = tcx.sess.source_map().lookup_char_pos(topmost.lo());
-            let const_loc = tcx.const_caller_location((
-                Symbol::intern(&caller.file.name.prefer_remapped().to_string_lossy()),
-                caller.line as u32,
-                caller.col_display as u32 + 1,
-            ));
+        self.mir.caller_location_span(source_info, self.caller_location, bx.tcx(), |span: Span| {
+            let const_loc = bx.tcx().span_as_caller_location(span);
             OperandRef::from_const(bx, const_loc, bx.tcx().caller_location_ty())
-        };
-
-        // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
-        // If so, the starting `source_info.span` is in the innermost inlined
-        // function, and will be replaced with outer callsite spans as long
-        // as the inlined functions were `#[track_caller]`.
-        loop {
-            let scope_data = &self.mir.source_scopes[source_info.scope];
-
-            if let Some((callee, callsite_span)) = scope_data.inlined {
-                // Stop inside the most nested non-`#[track_caller]` function,
-                // before ever reaching its caller (which is irrelevant).
-                if !callee.def.requires_caller_location(tcx) {
-                    return span_to_caller_location(source_info.span);
-                }
-                source_info.span = callsite_span;
-            }
-
-            // Skip past all of the parents with `inlined: None`.
-            match scope_data.inlined_parent_scope {
-                Some(parent) => source_info.scope = parent,
-                None => break,
-            }
-        }
-
-        // No inlined `SourceScope`s, or all of them were `#[track_caller]`.
-        self.caller_location.unwrap_or_else(|| span_to_caller_location(source_info.span))
+        })
     }
 
     fn get_personality_slot(&mut self, bx: &mut Bx) -> PlaceRef<'tcx, Bx::Value> {
@@ -1559,7 +1520,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     }
 
     fn terminate_block(&mut self, reason: UnwindTerminateReason) -> Bx::BasicBlock {
-        if let Some((cached_bb, cached_reason)) = self.terminate_block && reason == cached_reason {
+        if let Some((cached_bb, cached_reason)) = self.terminate_block
+            && reason == cached_reason
+        {
             return cached_bb;
         }
 
@@ -1627,7 +1590,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let fn_ty = bx.fn_decl_backend_type(&fn_abi);
 
         let llret = bx.call(fn_ty, None, Some(&fn_abi), fn_ptr, &[], funclet.as_ref());
-        bx.do_not_inline(llret);
+        bx.apply_attrs_to_cleanup_callsite(llret);
 
         bx.unreachable();
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs
index fde4e85f9..558f64fff 100644
--- a/compiler/rustc_codegen_ssa/src/mir/constant.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs
@@ -21,6 +21,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     }
 
     pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue<'tcx> {
+        // `MirUsedCollector` visited all constants before codegen began, so if we got here there
+        // can be no more constants that fail to evaluate.
         self.monomorphize(constant.const_)
             .eval(self.cx.tcx(), ty::ParamEnv::reveal_all(), Some(constant.span))
             .expect("erroneous constant not captured by required_consts")
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 8efef4405..136d06d56 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -117,9 +117,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     sym::vtable_size => {
                         let size_bound = bx.data_layout().ptr_sized_integer().signed_max() as u128;
                         bx.range_metadata(value, WrappingRange { start: 0, end: size_bound });
-                    },
+                    }
                     // Alignment is always nonzero.
-                    sym::vtable_align => bx.range_metadata(value, WrappingRange { start: 1, end: !0 }),
+                    sym::vtable_align => {
+                        bx.range_metadata(value, WrappingRange { start: 1, end: !0 })
+                    }
                     _ => {}
                 }
                 value
@@ -220,9 +222,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         } else {
                             bx.exactudiv(args[0].immediate(), args[1].immediate())
                         }
-                    },
+                    }
                     None => {
-                        bx.tcx().sess.emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
+                        bx.tcx().sess.emit_err(InvalidMonomorphization::BasicIntegerType {
+                            span,
+                            name,
+                            ty,
+                        });
                         return;
                     }
                 }
@@ -238,7 +244,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         _ => bug!(),
                     },
                     None => {
-                        bx.tcx().sess.emit_err(InvalidMonomorphization::BasicFloatType { span, name, ty: arg_tys[0] });
+                        bx.tcx().sess.emit_err(InvalidMonomorphization::BasicFloatType {
+                            span,
+                            name,
+                            ty: arg_tys[0],
+                        });
                         return;
                     }
                 }
@@ -246,11 +256,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             sym::float_to_int_unchecked => {
                 if float_type_width(arg_tys[0]).is_none() {
-                    bx.tcx().sess.emit_err(InvalidMonomorphization::FloatToIntUnchecked { span, ty: arg_tys[0] });
+                    bx.tcx().sess.emit_err(InvalidMonomorphization::FloatToIntUnchecked {
+                        span,
+                        ty: arg_tys[0],
+                    });
                     return;
                 }
                 let Some((_width, signed)) = int_type_width_signed(ret_ty, bx.tcx()) else {
-                    bx.tcx().sess.emit_err(InvalidMonomorphization::FloatToIntUnchecked { span, ty: ret_ty });
+                    bx.tcx().sess.emit_err(InvalidMonomorphization::FloatToIntUnchecked {
+                        span,
+                        ty: ret_ty,
+                    });
                     return;
                 };
                 if signed {
@@ -299,7 +315,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 };
 
                 let invalid_monomorphization = |ty| {
-                    bx.tcx().sess.emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
+                    bx.tcx().sess.emit_err(InvalidMonomorphization::BasicIntegerType {
+                        span,
+                        name,
+                        ty,
+                    });
                 };
 
                 match instruction {
@@ -319,7 +339,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 cmp = bx.ptrtoint(cmp, bx.type_isize());
                                 src = bx.ptrtoint(src, bx.type_isize());
                             }
-                            let pair = bx.atomic_cmpxchg(dst, cmp, src, parse_ordering(bx, success), parse_ordering(bx, failure), weak);
+                            let pair = bx.atomic_cmpxchg(
+                                dst,
+                                cmp,
+                                src,
+                                parse_ordering(bx, success),
+                                parse_ordering(bx, failure),
+                                weak,
+                            );
                             let val = bx.extract_value(pair, 0);
                             let success = bx.extract_value(pair, 1);
                             let val = bx.from_immediate(val);
@@ -345,11 +372,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 // Some platforms do not support atomic operations on pointers,
                                 // so we cast to integer first...
                                 let llty = bx.type_isize();
-                                let result = bx.atomic_load(llty, source, parse_ordering(bx, ordering), size);
+                                let result = bx.atomic_load(
+                                    llty,
+                                    source,
+                                    parse_ordering(bx, ordering),
+                                    size,
+                                );
                                 // ... and then cast the result back to a pointer
                                 bx.inttoptr(result, bx.backend_type(layout))
                             } else {
-                                bx.atomic_load(bx.backend_type(layout), source, parse_ordering(bx, ordering), size)
+                                bx.atomic_load(
+                                    bx.backend_type(layout),
+                                    source,
+                                    parse_ordering(bx, ordering),
+                                    size,
+                                )
                             }
                         } else {
                             return invalid_monomorphization(ty);
@@ -375,12 +412,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     }
 
                     "fence" => {
-                        bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::CrossThread);
+                        bx.atomic_fence(
+                            parse_ordering(bx, ordering),
+                            SynchronizationScope::CrossThread,
+                        );
                         return;
                     }
 
                     "singlethreadfence" => {
-                        bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::SingleThread);
+                        bx.atomic_fence(
+                            parse_ordering(bx, ordering),
+                            SynchronizationScope::SingleThread,
+                        );
                         return;
                     }
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index a61018f98..d0b799e08 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -209,18 +209,11 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         caller_location: None,
     };
 
-    fx.per_local_var_debug_info = fx.compute_per_local_var_debug_info(&mut start_bx);
+    // It may seem like we should iterate over `required_consts` to ensure they all successfully
+    // evaluate; however, the `MirUsedCollector` already did that during the collection phase of
+    // monomorphization so we don't have to do it again.
 
-    // Rust post-monomorphization checks; we later rely on them.
-    if let Err(err) =
-        mir.post_mono_checks(cx.tcx(), ty::ParamEnv::reveal_all(), |c| Ok(fx.monomorphize(c)))
-    {
-        err.emit_err(cx.tcx());
-        // This IR shouldn't ever be emitted, but let's try to guard against any of this code
-        // ever running.
-        start_bx.abort();
-        return;
-    }
+    fx.per_local_var_debug_info = fx.compute_per_local_var_debug_info(&mut start_bx);
 
     let memory_locals = analyze::non_ssa_locals(&fx);
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index fc8d33891..8e5019967 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -13,7 +13,7 @@ use rustc_middle::ty::cast::{CastTy, IntTy};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, adjustment::PointerCoercion, Instance, Ty, TyCtxt};
 use rustc_session::config::OptLevel;
-use rustc_span::source_map::{Span, DUMMY_SP};
+use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::{self, FIRST_VARIANT};
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
@@ -239,17 +239,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 };
                 if let OperandValueKind::Immediate(out_scalar) = cast_kind
                     && in_scalar.size(self.cx) == out_scalar.size(self.cx)
-                        {
-                            let operand_bty = bx.backend_type(operand.layout);
-                            let cast_bty = bx.backend_type(cast);
-                            Some(OperandValue::Immediate(self.transmute_immediate(
-                                bx,
-                                imm,
-                                in_scalar,
-                                operand_bty,
-                                out_scalar,
-                                cast_bty,
-                            )))
+                {
+                    let operand_bty = bx.backend_type(operand.layout);
+                    let cast_bty = bx.backend_type(cast);
+                    Some(OperandValue::Immediate(self.transmute_immediate(
+                        bx,
+                        imm,
+                        in_scalar,
+                        operand_bty,
+                        out_scalar,
+                        cast_bty,
+                    )))
                 } else {
                     None
                 }
@@ -680,7 +680,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         layout.align.abi.bytes()
                     }
                     mir::NullOp::OffsetOf(fields) => {
-                        layout.offset_of_subfield(bx.cx(), fields.iter().map(|f| f.index())).bytes()
+                        layout.offset_of_subfield(bx.cx(), fields.iter()).bytes()
                     }
                 };
                 let val = bx.cx().const_usize(val);
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index baf6b19d3..2936f1de3 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -23,6 +23,15 @@ pub const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"];
 // check whether they're named already elsewhere in rust
 // e.g. in stdarch and whether the given name matches LLVM's
 // if it doesn't, to_llvm_feature in llvm_util in rustc_codegen_llvm needs to be adapted
+//
+// When adding a new feature, be particularly mindful of features that affect function ABIs. Those
+// need to be treated very carefully to avoid introducing unsoundness! This often affects features
+// that enable/disable hardfloat support (see https://github.com/rust-lang/rust/issues/116344 for an
+// example of this going wrong), but features enabling new SIMD registers are also a concern (see
+// https://github.com/rust-lang/rust/issues/116558 for an example of this going wrong).
+//
+// Stabilizing a target feature (setting the 2nd component of the pair to `None`) requires t-lang
+// approval.
 
 const ARM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     // tidy-alphabetical-start
@@ -244,38 +253,38 @@ const MIPS_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
 
 const RISCV_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     // tidy-alphabetical-start
-    ("a", Some(sym::riscv_target_feature)),
-    ("c", Some(sym::riscv_target_feature)),
+    ("a", None),
+    ("c", None),
     ("d", Some(sym::riscv_target_feature)),
     ("e", Some(sym::riscv_target_feature)),
     ("f", Some(sym::riscv_target_feature)),
-    ("m", Some(sym::riscv_target_feature)),
+    ("m", None),
     ("relax", Some(sym::riscv_target_feature)),
     ("unaligned-scalar-mem", Some(sym::riscv_target_feature)),
     ("v", Some(sym::riscv_target_feature)),
-    ("zba", Some(sym::riscv_target_feature)),
-    ("zbb", Some(sym::riscv_target_feature)),
-    ("zbc", Some(sym::riscv_target_feature)),
-    ("zbkb", Some(sym::riscv_target_feature)),
-    ("zbkc", Some(sym::riscv_target_feature)),
-    ("zbkx", Some(sym::riscv_target_feature)),
-    ("zbs", Some(sym::riscv_target_feature)),
+    ("zba", None),
+    ("zbb", None),
+    ("zbc", None),
+    ("zbkb", None),
+    ("zbkc", None),
+    ("zbkx", None),
+    ("zbs", None),
     ("zdinx", Some(sym::riscv_target_feature)),
     ("zfh", Some(sym::riscv_target_feature)),
     ("zfhmin", Some(sym::riscv_target_feature)),
     ("zfinx", Some(sym::riscv_target_feature)),
     ("zhinx", Some(sym::riscv_target_feature)),
     ("zhinxmin", Some(sym::riscv_target_feature)),
-    ("zk", Some(sym::riscv_target_feature)),
-    ("zkn", Some(sym::riscv_target_feature)),
-    ("zknd", Some(sym::riscv_target_feature)),
-    ("zkne", Some(sym::riscv_target_feature)),
-    ("zknh", Some(sym::riscv_target_feature)),
-    ("zkr", Some(sym::riscv_target_feature)),
-    ("zks", Some(sym::riscv_target_feature)),
-    ("zksed", Some(sym::riscv_target_feature)),
-    ("zksh", Some(sym::riscv_target_feature)),
-    ("zkt", Some(sym::riscv_target_feature)),
+    ("zk", None),
+    ("zkn", None),
+    ("zknd", None),
+    ("zkne", None),
+    ("zknh", None),
+    ("zkr", None),
+    ("zks", None),
+    ("zksed", None),
+    ("zksh", None),
+    ("zkt", None),
     // tidy-alphabetical-end
 ];
 
@@ -342,6 +351,19 @@ const CSKY_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("hard-float-abi", Some(sym::csky_target_feature)),
     // tidy-alphabetical-end
 ];
+
+const LOONGARCH_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
+    // tidy-alphabetical-start
+    ("d", Some(sym::loongarch_target_feature)),
+    ("f", Some(sym::loongarch_target_feature)),
+    ("lasx", Some(sym::loongarch_target_feature)),
+    ("lbt", Some(sym::loongarch_target_feature)),
+    ("lsx", Some(sym::loongarch_target_feature)),
+    ("lvz", Some(sym::loongarch_target_feature)),
+    ("ual", Some(sym::loongarch_target_feature)),
+    // tidy-alphabetical-end
+];
+
 /// When rustdoc is running, provide a list of all known features so that all their respective
 /// primitives may be documented.
 ///
@@ -358,6 +380,7 @@ pub fn all_known_features() -> impl Iterator<Item = (&'static str, Option<Symbol
         .chain(WASM_ALLOWED_FEATURES.iter())
         .chain(BPF_ALLOWED_FEATURES.iter())
         .chain(CSKY_ALLOWED_FEATURES)
+        .chain(LOONGARCH_ALLOWED_FEATURES)
         .cloned()
 }
 
@@ -373,6 +396,7 @@ pub fn supported_target_features(sess: &Session) -> &'static [(&'static str, Opt
         "wasm32" | "wasm64" => WASM_ALLOWED_FEATURES,
         "bpf" => BPF_ALLOWED_FEATURES,
         "csky" => CSKY_ALLOWED_FEATURES,
+        "loongarch64" => LOONGARCH_ALLOWED_FEATURES,
         _ => &[],
     }
 }
@@ -445,6 +469,7 @@ pub fn from_target_feature(
                 Some(sym::bpf_target_feature) => rust_features.bpf_target_feature,
                 Some(sym::aarch64_ver_target_feature) => rust_features.aarch64_ver_target_feature,
                 Some(sym::csky_target_feature) => rust_features.csky_target_feature,
+                Some(sym::loongarch_target_feature) => rust_features.loongarch_target_feature,
                 Some(name) => bug!("unknown target feature gate {}", name),
                 None => true,
             };
diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs
index ac8123bc1..35744d9a1 100644
--- a/compiler/rustc_codegen_ssa/src/traits/backend.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs
@@ -104,11 +104,7 @@ pub trait CodegenBackend {
         outputs: &OutputFilenames,
     ) -> Result<(CodegenResults, FxIndexMap<WorkProductId, WorkProduct>), ErrorGuaranteed>;
 
-    /// This is called on the returned `Box<dyn Any>` from `join_codegen`
-    ///
-    /// # Panics
-    ///
-    /// Panics when the passed `Box<dyn Any>` was not returned by `join_codegen`.
+    /// This is called on the returned `CodegenResults` from `join_codegen`
     fn link(
         &self,
         sess: &Session,
diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs
index 853c6934c..aa411f002 100644
--- a/compiler/rustc_codegen_ssa/src/traits/builder.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs
@@ -332,5 +332,5 @@ pub trait BuilderMethods<'a, 'tcx>:
     ) -> Self::Value;
     fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
 
-    fn do_not_inline(&mut self, llret: Self::Value);
+    fn apply_attrs_to_cleanup_callsite(&mut self, llret: Self::Value);
 }
diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs
index dc3dbd9d8..b1fde8e4d 100644
--- a/compiler/rustc_codegen_ssa/src/traits/type_.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs
@@ -30,7 +30,7 @@ pub trait BaseTypeMethods<'tcx>: Backend<'tcx> {
     fn type_ptr_ext(&self, address_space: AddressSpace) -> Self::Type;
     fn element_type(&self, ty: Self::Type) -> Self::Type;
 
-    /// Returns the number of elements in `self` if it is a LLVM vector type.
+    /// Returns the number of elements in `self` if it is an LLVM vector type.
     fn vector_length(&self, ty: Self::Type) -> usize;
 
     fn float_width(&self, ty: Self::Type) -> usize;