Merging upstream version 1.69.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

24 files changed, 387 insertions, 515 deletions

diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml
index 0d2d2ec68..c55991e00 100644
--- a/compiler/rustc_codegen_ssa/Cargo.toml
+++ b/compiler/rustc_codegen_ssa/Cargo.toml
@@ -41,7 +41,6 @@ rustc_metadata = { path = "../rustc_metadata" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_target = { path = "../rustc_target" }
 rustc_session = { path = "../rustc_session" }
-rustc_const_eval = { path = "../rustc_const_eval" }
 
 [dependencies.object]
 version = "0.30.1"
diff --git a/compiler/rustc_error_messages/locales/en-US/codegen_ssa.ftl b/compiler/rustc_codegen_ssa/locales/en-US.ftl
index c8c7afb5f..8fe5f8d50 100644
--- a/compiler/rustc_error_messages/locales/en-US/codegen_ssa.ftl
+++ b/compiler/rustc_codegen_ssa/locales/en-US.ftl
@@ -22,7 +22,7 @@ codegen_ssa_ignoring_output = ignoring -o because multiple .{$extension} files w
 
 codegen_ssa_create_temp_dir = couldn't create a temp dir: {$error}
 
-codegen_ssa_incompatible_linking_modifiers = the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs
+codegen_ssa_incompatible_linking_modifiers = link modifiers combination `+bundle,+whole-archive` is unstable when generating rlibs
 
 codegen_ssa_add_native_library = failed to add native library {$library_path}: {$error}
 
@@ -179,9 +179,9 @@ codegen_ssa_extract_bundled_libs_write_file = failed to write file '{$rlib}': {$
 
 codegen_ssa_unsupported_arch = unsupported arch `{$arch}` for os `{$os}`
 
-codegen_ssa_apple_sdk_error_sdk_path = failed to get {$sdk_name} SDK path: {error}
+codegen_ssa_apple_sdk_error_sdk_path = failed to get {$sdk_name} SDK path: {$error}
 
-codegen_ssa_read_file = failed to read file: {message}
+codegen_ssa_read_file = failed to read file: {$message}
 
 codegen_ssa_unsupported_link_self_contained = option `-C link-self-contained` is not supported on this target
 
diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs
index d3cd085cf..66ec8f5f5 100644
--- a/compiler/rustc_codegen_ssa/src/back/archive.rs
+++ b/compiler/rustc_codegen_ssa/src/back/archive.rs
@@ -203,7 +203,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
             }
         }
 
-        self.src_archives.push((archive_path.to_owned(), archive_map));
+        self.src_archives.push((archive_path, archive_map));
         Ok(())
     }
 
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 34e042376..8bb143ed3 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -270,10 +270,9 @@ pub fn each_linked_rlib(
 
 /// Create an 'rlib'.
 ///
-/// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains
-/// the object file of the crate, but it also contains all of the object files from native
-/// libraries. This is done by unzipping native libraries and inserting all of the contents into
-/// this archive.
+/// An rlib in its current incarnation is essentially a renamed .a file (with "dummy" object files).
+/// The rlib primarily contains the object file of the crate, but it also some of the object files
+/// from native libraries.
 fn link_rlib<'a>(
     sess: &'a Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
@@ -347,44 +346,23 @@ fn link_rlib<'a>(
     // loaded from the libraries found here and then encode that into the
     // metadata of the rlib we're generating somehow.
     for lib in codegen_results.crate_info.used_libraries.iter() {
-        match lib.kind {
-            NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
-                if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.packed_bundled_libs => {}
-            NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
-                if flavor == RlibFlavor::Normal =>
-            {
-                // Don't allow mixing +bundle with +whole_archive since an rlib may contain
-                // multiple native libs, some of which are +whole-archive and some of which are
-                // -whole-archive and it isn't clear how we can currently handle such a
-                // situation correctly.
-                // See https://github.com/rust-lang/rust/issues/88085#issuecomment-901050897
-                sess.emit_err(errors::IncompatibleLinkingModifiers);
-            }
-            NativeLibKind::Static { bundle: None | Some(true), .. } => {}
-            NativeLibKind::Static { bundle: Some(false), .. }
-            | NativeLibKind::Dylib { .. }
-            | NativeLibKind::Framework { .. }
-            | NativeLibKind::RawDylib
-            | NativeLibKind::LinkArg
-            | NativeLibKind::Unspecified => continue,
-        }
-        if let Some(name) = lib.name {
-            let location =
+        let NativeLibKind::Static { bundle: None | Some(true), whole_archive } = lib.kind else {
+            continue;
+        };
+        if whole_archive == Some(true) && !codegen_results.crate_info.feature_packed_bundled_libs {
+            sess.emit_err(errors::IncompatibleLinkingModifiers);
+        }
+        if flavor == RlibFlavor::Normal && let Some(filename) = lib.filename {
+            let path = find_native_static_library(filename.as_str(), true, &lib_search_paths, sess);
+            let src = read(path).map_err(|e| sess.emit_fatal(errors::ReadFileError {message: e }))?;
+            let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
+            let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
+            packed_bundled_libs.push(wrapper_file);
+        } else if let Some(name) = lib.name {
+            let path =
                 find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
-            if sess.opts.unstable_opts.packed_bundled_libs && flavor == RlibFlavor::Normal {
-                let filename = lib.filename.unwrap();
-                let lib_path =
-                    find_native_static_library(filename.as_str(), true, &lib_search_paths, sess);
-                let src = read(lib_path)
-                    .map_err(|e| sess.emit_fatal(errors::ReadFileError { message: e }))?;
-                let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
-                let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
-                packed_bundled_libs.push(wrapper_file);
-                continue;
-            }
-            ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|error| {
-                sess.emit_fatal(errors::AddNativeLibrary { library_path: location, error });
-            });
+            ab.add_archive(&path, Box::new(|_| false)).unwrap_or_else(|error| {
+                sess.emit_fatal(errors::AddNativeLibrary { library_path: path, error })});
         }
     }
 
@@ -516,36 +494,14 @@ fn link_staticlib<'a>(
         &codegen_results.crate_info,
         Some(CrateType::Staticlib),
         &mut |cnum, path| {
-            let name = codegen_results.crate_info.crate_name[&cnum];
-            let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
-
-            // Here when we include the rlib into our staticlib we need to make a
-            // decision whether to include the extra object files along the way.
-            // These extra object files come from statically included native
-            // libraries, but they may be cfg'd away with #[link(cfg(..))].
-            //
-            // This unstable feature, though, only needs liblibc to work. The only
-            // use case there is where musl is statically included in liblibc.rlib,
-            // so if we don't want the included version we just need to skip it. As
-            // a result the logic here is that if *any* linked library is cfg'd away
-            // we just skip all object files.
-            //
-            // Clearly this is not sufficient for a general purpose feature, and
-            // we'd want to read from the library's metadata to determine which
-            // object files come from where and selectively skip them.
-            let skip_object_files = native_libs.iter().any(|lib| {
-                matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
-                    && !relevant_lib(sess, lib)
-            });
-
             let lto = are_upstream_rust_objects_already_included(sess)
                 && !ignored_for_lto(sess, &codegen_results.crate_info, cnum);
 
-            // Ignoring obj file starting with the crate name
-            // as simple comparison is not enough - there
-            // might be also an extra name suffix
-            let obj_start = name.as_str().to_owned();
+            let native_libs = codegen_results.crate_info.native_libraries[&cnum].iter();
+            let relevant = native_libs.clone().filter(|lib| relevant_lib(sess, &lib));
+            let relevant_libs: FxHashSet<_> = relevant.filter_map(|lib| lib.filename).collect();
 
+            let bundled_libs: FxHashSet<_> = native_libs.filter_map(|lib| lib.filename).collect();
             ab.add_archive(
                 path,
                 Box::new(move |fname: &str| {
@@ -559,20 +515,25 @@ fn link_staticlib<'a>(
                         return true;
                     }
 
-                    // Otherwise if this is *not* a rust object and we're skipping
-                    // objects then skip this file
-                    if skip_object_files
-                        && (!fname.starts_with(&obj_start) || !fname.ends_with(".o"))
-                    {
+                    // Skip objects for bundled libs.
+                    if bundled_libs.contains(&Symbol::intern(fname)) {
                         return true;
                     }
 
-                    // ok, don't skip this
                     false
                 }),
             )
             .unwrap();
 
+            archive_builder_builder
+                .extract_bundled_libs(path, tempdir.as_ref(), &relevant_libs)
+                .unwrap_or_else(|e| sess.emit_fatal(e));
+            for filename in relevant_libs {
+                let joined = tempdir.as_ref().join(filename.as_str());
+                let path = joined.as_path();
+                ab.add_archive(path, Box::new(|_| false)).unwrap();
+            }
+
             all_native_libs
                 .extend(codegen_results.crate_info.native_libraries[&cnum].iter().cloned());
         },
@@ -599,7 +560,8 @@ fn link_dwarf_object<'a>(
     cg_results: &CodegenResults,
     executable_out_filename: &Path,
 ) {
-    let dwp_out_filename = executable_out_filename.with_extension("dwp");
+    let mut dwp_out_filename = executable_out_filename.to_path_buf().into_os_string();
+    dwp_out_filename.push(".dwp");
     debug!(?dwp_out_filename, ?executable_out_filename);
 
     #[derive(Default)]
@@ -1302,12 +1264,6 @@ fn preserve_objects_for_their_debuginfo(sess: &Session) -> (bool, bool) {
         return (false, false);
     }
 
-    // If we're only producing artifacts that are archives, no need to preserve
-    // the objects as they're losslessly contained inside the archives.
-    if sess.crate_types().iter().all(|&x| x.is_archive()) {
-        return (false, false);
-    }
-
     match (sess.split_debuginfo(), sess.opts.unstable_opts.split_dwarf_kind) {
         // If there is no split debuginfo then do not preserve objects.
         (SplitDebuginfo::Off, _) => (false, false),
@@ -2070,7 +2026,7 @@ fn linker_with_args<'a>(
         .native_libraries
         .iter()
         .filter_map(|(cnum, libraries)| {
-            (dependency_linkage[cnum.as_usize() - 1] != Linkage::Static).then(|| libraries)
+            (dependency_linkage[cnum.as_usize() - 1] != Linkage::Static).then_some(libraries)
         })
         .flatten();
     for (raw_dylib_name, raw_dylib_imports) in
@@ -2597,18 +2553,8 @@ fn add_static_crate<'a>(
         cmd.link_rlib(&fix_windows_verbatim_for_gcc(path));
     };
 
-    // See the comment above in `link_staticlib` and `link_rlib` for why if
-    // there's a static library that's not relevant we skip all object
-    // files.
-    let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
-    let skip_native = native_libs.iter().any(|lib| {
-        matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
-            && !relevant_lib(sess, lib)
-    });
-
-    if (!are_upstream_rust_objects_already_included(sess)
-        || ignored_for_lto(sess, &codegen_results.crate_info, cnum))
-        && !skip_native
+    if !are_upstream_rust_objects_already_included(sess)
+        || ignored_for_lto(sess, &codegen_results.crate_info, cnum)
     {
         link_upstream(cratepath);
         return;
@@ -2639,17 +2585,13 @@ fn add_static_crate<'a>(
                 let is_rust_object =
                     canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f);
 
-                // If we've been requested to skip all native object files
-                // (those not generated by the rust compiler) then we can skip
-                // this file. See above for why we may want to do this.
-                let skip_because_cfg_say_so = skip_native && !is_rust_object;
-
                 // If we're performing LTO and this is a rust-generated object
                 // file, then we don't need the object file as it's part of the
                 // LTO module. Note that `#![no_builtins]` is excluded from LTO,
                 // though, so we let that object file slide.
-                let skip_because_lto =
-                    upstream_rust_objects_already_included && is_rust_object && is_builtins;
+                if upstream_rust_objects_already_included && is_rust_object && is_builtins {
+                    return true;
+                }
 
                 // We skip native libraries because:
                 // 1. This native libraries won't be used from the generated rlib,
@@ -2660,10 +2602,6 @@ fn add_static_crate<'a>(
                     return true;
                 }
 
-                if skip_because_cfg_say_so || skip_because_lto {
-                    return true;
-                }
-
                 false
             }),
         ) {
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index eaf1e9817..52c01b423 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -473,13 +473,13 @@ impl<'a> Linker for GccLinker<'a> {
         self.cmd.arg(path);
     }
     fn full_relro(&mut self) {
-        self.linker_args(&["-zrelro", "-znow"]);
+        self.linker_args(&["-z", "relro", "-z", "now"]);
     }
     fn partial_relro(&mut self) {
-        self.linker_arg("-zrelro");
+        self.linker_args(&["-z", "relro"]);
     }
     fn no_relro(&mut self) {
-        self.linker_arg("-znorelro");
+        self.linker_args(&["-z", "norelro"]);
     }
 
     fn link_rust_dylib(&mut self, lib: &str, _path: &Path) {
@@ -758,7 +758,7 @@ impl<'a> Linker for GccLinker<'a> {
         if self.sess.target.is_like_windows {
             self.linker_arg("--nxcompat");
         } else if self.is_gnu {
-            self.linker_arg("-znoexecstack");
+            self.linker_args(&["-z", "noexecstack"]);
         }
     }
 
@@ -1364,16 +1364,16 @@ impl<'a> Linker for L4Bender<'a> {
     }
 
     fn full_relro(&mut self) {
-        self.cmd.arg("-zrelro");
-        self.cmd.arg("-znow");
+        self.cmd.arg("-z").arg("relro");
+        self.cmd.arg("-z").arg("now");
     }
 
     fn partial_relro(&mut self) {
-        self.cmd.arg("-zrelro");
+        self.cmd.arg("-z").arg("relro");
     }
 
     fn no_relro(&mut self) {
-        self.cmd.arg("-znorelro");
+        self.cmd.arg("-z").arg("norelro");
     }
 
     fn cmd(&mut self) -> &mut Command {
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index 7d3c14fec..019ec0758 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -33,6 +33,7 @@ use rustc_target::spec::{RelocModel, Target};
 /// <dt>dylib</dt>
 /// <dd>The metadata can be found in the `.rustc` section of the shared library.</dd>
 /// </dl>
+#[derive(Debug)]
 pub struct DefaultMetadataLoader;
 
 fn load_metadata_with(
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index 57a99e74c..067a3e167 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -2,9 +2,8 @@ use std::collections::hash_map::Entry::*;
 
 use rustc_ast::expand::allocator::ALLOCATOR_METHODS;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LOCAL_CRATE};
-use rustc_hir::Node;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::middle::exported_symbols::{
     metadata_symbol_name, ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel,
@@ -12,7 +11,7 @@ use rustc_middle::middle::exported_symbols::{
 use rustc_middle::ty::query::{ExternProviders, Providers};
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
 use rustc_middle::ty::Instance;
-use rustc_middle::ty::{self, SymbolName, TyCtxt};
+use rustc_middle::ty::{self, DefIdTree, SymbolName, TyCtxt};
 use rustc_session::config::{CrateType, OomStrategy};
 use rustc_target::spec::SanitizerSet;
 
@@ -74,32 +73,34 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
             //
             // As a result, if this id is an FFI item (foreign item) then we only
             // let it through if it's included statically.
-            match tcx.hir().get_by_def_id(def_id) {
-                Node::ForeignItem(..) => {
-                    tcx.native_library(def_id).map_or(false, |library| library.kind.is_statically_included()).then_some(def_id)
-                }
+            if let Some(parent_id) = tcx.opt_local_parent(def_id)
+                && let DefKind::ForeignMod = tcx.def_kind(parent_id)
+            {
+                let library = tcx.native_library(def_id)?;
+                return library.kind.is_statically_included().then_some(def_id);
+            }
 
-                // Only consider nodes that actually have exported symbols.
-                Node::Item(&hir::Item {
-                    kind: hir::ItemKind::Static(..) | hir::ItemKind::Fn(..),
-                    ..
-                })
-                | Node::ImplItem(&hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }) => {
-                    let generics = tcx.generics_of(def_id);
-                    if !generics.requires_monomorphization(tcx)
-                        // Functions marked with #[inline] are codegened with "internal"
-                        // linkage and are not exported unless marked with an extern
-                        // indicator
-                        && (!Instance::mono(tcx, def_id.to_def_id()).def.generates_cgu_internal_copy(tcx)
-                            || tcx.codegen_fn_attrs(def_id.to_def_id()).contains_extern_indicator())
-                    {
-                        Some(def_id)
-                    } else {
-                        None
-                    }
-                }
+            // Only consider nodes that actually have exported symbols.
+            match tcx.def_kind(def_id) {
+                DefKind::Fn | DefKind::Static(_) => {}
+                DefKind::AssocFn if tcx.impl_of_method(def_id.to_def_id()).is_some() => {}
+                _ => return None,
+            };
 
-                _ => None,
+            let generics = tcx.generics_of(def_id);
+            if generics.requires_monomorphization(tcx) {
+                return None;
+            }
+
+            // Functions marked with #[inline] are codegened with "internal"
+            // linkage and are not exported unless marked with an extern
+            // indicator
+            if !Instance::mono(tcx, def_id.to_def_id()).def.generates_cgu_internal_copy(tcx)
+                || tcx.codegen_fn_attrs(def_id.to_def_id()).contains_extern_indicator()
+            {
+                Some(def_id)
+            } else {
+                None
             }
         })
         .map(|def_id| {
@@ -118,7 +119,7 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
                 tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())),
                 export_level
             );
-            (def_id.to_def_id(), SymbolExportInfo {
+            let info = SymbolExportInfo {
                 level: export_level,
                 kind: if tcx.is_static(def_id.to_def_id()) {
                     if codegen_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) {
@@ -130,8 +131,10 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
                     SymbolExportKind::Text
                 },
                 used: codegen_attrs.flags.contains(CodegenFnAttrFlags::USED)
-                    || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) || used,
-            })
+                    || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)
+                    || used,
+            };
+            (def_id.to_def_id(), info)
         })
         .collect();
 
@@ -370,7 +373,7 @@ fn upstream_monomorphizations_provider(
                 ExportedSymbol::Generic(def_id, substs) => (def_id, substs),
                 ExportedSymbol::DropGlue(ty) => {
                     if let Some(drop_in_place_fn_def_id) = drop_in_place_fn_def_id {
-                        (drop_in_place_fn_def_id, tcx.intern_substs(&[ty.into()]))
+                        (drop_in_place_fn_def_id, tcx.mk_substs(&[ty.into()]))
                     } else {
                         // `drop_in_place` in place does not exist, don't try
                         // to use it.
@@ -457,9 +460,7 @@ fn symbol_export_level(tcx: TyCtxt<'_>, sym_def_id: DefId) -> SymbolExportLevel
         let target = &tcx.sess.target.llvm_target;
         // WebAssembly cannot export data symbols, so reduce their export level
         if target.contains("emscripten") {
-            if let Some(Node::Item(&hir::Item { kind: hir::ItemKind::Static(..), .. })) =
-                tcx.hir().get_if_local(sym_def_id)
-            {
+            if let DefKind::Static(_) = tcx.def_kind(sym_def_id) {
                 return SymbolExportLevel::Rust;
             }
         }
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 9f1614af7..8508ab875 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -305,8 +305,12 @@ impl TargetMachineFactoryConfig {
 }
 
 pub type TargetMachineFactoryFn<B> = Arc<
-    dyn Fn(TargetMachineFactoryConfig) -> Result<<B as WriteBackendMethods>::TargetMachine, String>
-        + Send
+    dyn Fn(
+            TargetMachineFactoryConfig,
+        ) -> Result<
+            <B as WriteBackendMethods>::TargetMachine,
+            <B as WriteBackendMethods>::TargetMachineError,
+        > + Send
         + Sync,
 >;
 
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 32d3cfe6f..73179249b 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -39,7 +39,7 @@ use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::Symbol;
 use rustc_span::{DebuggerVisualizerFile, DebuggerVisualizerType};
-use rustc_target::abi::{Align, Size, VariantIdx};
+use rustc_target::abi::{Align, VariantIdx};
 
 use std::collections::BTreeSet;
 use std::time::{Duration, Instant};
@@ -148,7 +148,7 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         cx.tcx().struct_lockstep_tails_erasing_lifetimes(source, target, bx.param_env());
     match (source.kind(), target.kind()) {
         (&ty::Array(_, len), &ty::Slice(_)) => {
-            cx.const_usize(len.eval_usize(cx.tcx(), ty::ParamEnv::reveal_all()))
+            cx.const_usize(len.eval_target_usize(cx.tcx(), ty::ParamEnv::reveal_all()))
         }
         (
             &ty::Dynamic(ref data_a, _, src_dyn_kind),
@@ -273,12 +273,13 @@ pub fn cast_to_dyn_star<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         matches!(dst_ty.kind(), ty::Dynamic(_, _, ty::DynStar)),
         "destination type must be a dyn*"
     );
-    // FIXME(dyn-star): this is probably not the best way to check if this is
-    // a pointer, and really we should ensure that the value is a suitable
-    // pointer earlier in the compilation process.
-    let src = match src_ty_and_layout.pointee_info_at(bx.cx(), Size::ZERO) {
-        Some(_) => bx.ptrtoint(src, bx.cx().type_isize()),
-        None => bx.bitcast(src, bx.type_isize()),
+    // FIXME(dyn-star): We can remove this when all supported LLVMs use opaque ptrs only.
+    let unit_ptr = bx.cx().type_ptr_to(bx.cx().type_struct(&[], false));
+    let src = match bx.cx().type_kind(bx.cx().backend_type(src_ty_and_layout)) {
+        TypeKind::Pointer => bx.pointercast(src, unit_ptr),
+        TypeKind::Integer => bx.inttoptr(src, unit_ptr),
+        // FIXME(dyn-star): We probably have to do a bitcast first, then inttoptr.
+        kind => bug!("unexpected TypeKind for left-hand side of `dyn*` cast: {kind:?}"),
     };
     (src, unsized_info(bx, src_ty_and_layout.ty, dst_ty, old_info))
 }
@@ -436,7 +437,7 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             cx.type_func(&[], cx.type_int())
         };
 
-        let main_ret_ty = cx.tcx().fn_sig(rust_main_def_id).output();
+        let main_ret_ty = cx.tcx().fn_sig(rust_main_def_id).no_bound_vars().unwrap().output();
         // Given that `main()` has no arguments,
         // then its return type cannot have
         // late-bound regions, since late-bound
@@ -475,7 +476,7 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                     cx.tcx(),
                     ty::ParamEnv::reveal_all(),
                     start_def_id,
-                    cx.tcx().intern_substs(&[main_ret_ty.into()]),
+                    cx.tcx().mk_substs(&[main_ret_ty.into()]),
                 )
                 .unwrap()
                 .unwrap(),
@@ -579,7 +580,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
         }
     }
 
-    let metadata_module = if need_metadata_module {
+    let metadata_module = need_metadata_module.then(|| {
         // Emit compressed metadata object.
         let metadata_cgu_name =
             cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata")).to_string();
@@ -594,17 +595,15 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
             if let Err(error) = std::fs::write(&file_name, data) {
                 tcx.sess.emit_fatal(errors::MetadataObjectFileWrite { error });
             }
-            Some(CompiledModule {
+            CompiledModule {
                 name: metadata_cgu_name,
                 kind: ModuleKind::Metadata,
                 object: Some(file_name),
                 dwarf_object: None,
                 bytecode: None,
-            })
+            }
         })
-    } else {
-        None
-    };
+    });
 
     let ongoing_codegen = start_async_codegen(
         backend.clone(),
@@ -858,6 +857,7 @@ impl CrateInfo {
             dependency_formats: tcx.dependency_formats(()).clone(),
             windows_subsystem,
             natvis_debugger_visualizers: Default::default(),
+            feature_packed_bundled_libs: tcx.features().packed_bundled_libs,
         };
         let crates = tcx.crates(());
 
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 8808ad2dc..7d5c00486 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -85,55 +85,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
         } else if attr.has_name(sym::rustc_allocator) {
             codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR;
         } else if attr.has_name(sym::ffi_returns_twice) {
-            if tcx.is_foreign_item(did) {
-                codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_RETURNS_TWICE;
-            } else {
-                // `#[ffi_returns_twice]` is only allowed `extern fn`s.
-                struct_span_err!(
-                    tcx.sess,
-                    attr.span,
-                    E0724,
-                    "`#[ffi_returns_twice]` may only be used on foreign functions"
-                )
-                .emit();
-            }
+            codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_RETURNS_TWICE;
         } else if attr.has_name(sym::ffi_pure) {
-            if tcx.is_foreign_item(did) {
-                if attrs.iter().any(|a| a.has_name(sym::ffi_const)) {
-                    // `#[ffi_const]` functions cannot be `#[ffi_pure]`
-                    struct_span_err!(
-                        tcx.sess,
-                        attr.span,
-                        E0757,
-                        "`#[ffi_const]` function cannot be `#[ffi_pure]`"
-                    )
-                    .emit();
-                } else {
-                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_PURE;
-                }
-            } else {
-                // `#[ffi_pure]` is only allowed on foreign functions
-                struct_span_err!(
-                    tcx.sess,
-                    attr.span,
-                    E0755,
-                    "`#[ffi_pure]` may only be used on foreign functions"
-                )
-                .emit();
-            }
+            codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_PURE;
         } else if attr.has_name(sym::ffi_const) {
-            if tcx.is_foreign_item(did) {
-                codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_CONST;
-            } else {
-                // `#[ffi_const]` is only allowed on foreign functions
-                struct_span_err!(
-                    tcx.sess,
-                    attr.span,
-                    E0756,
-                    "`#[ffi_const]` may only be used on foreign functions"
-                )
-                .emit();
-            }
+            codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_CONST;
         } else if attr.has_name(sym::rustc_nounwind) {
             codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND;
         } else if attr.has_name(sym::rustc_reallocator) {
@@ -214,7 +170,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
             }
         } else if attr.has_name(sym::cmse_nonsecure_entry) {
             if validate_fn_only_attr(attr.span)
-                && !matches!(tcx.fn_sig(did).abi(), abi::Abi::C { .. })
+                && !matches!(tcx.fn_sig(did).skip_binder().abi(), abi::Abi::C { .. })
             {
                 struct_span_err!(
                     tcx.sess,
@@ -234,7 +190,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
         } else if attr.has_name(sym::track_caller) {
             if !tcx.is_closure(did.to_def_id())
                 && validate_fn_only_attr(attr.span)
-                && tcx.fn_sig(did).abi() != abi::Abi::Rust
+                && tcx.fn_sig(did).skip_binder().abi() != abi::Abi::Rust
             {
                 struct_span_err!(tcx.sess, attr.span, E0737, "`#[track_caller]` requires Rust ABI")
                     .emit();
@@ -266,7 +222,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
             }
         } else if attr.has_name(sym::target_feature) {
             if !tcx.is_closure(did.to_def_id())
-                && tcx.fn_sig(did).unsafety() == hir::Unsafety::Normal
+                && tcx.fn_sig(did).skip_binder().unsafety() == hir::Unsafety::Normal
             {
                 if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc {
                     // The `#[target_feature]` attribute is allowed on
@@ -339,7 +295,8 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
             if let Some(list) = attr.meta_item_list() {
                 for item in list.iter() {
                     if item.has_name(sym::address) {
-                        codegen_fn_attrs.no_sanitize |= SanitizerSet::ADDRESS;
+                        codegen_fn_attrs.no_sanitize |=
+                            SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS;
                     } else if item.has_name(sym::cfi) {
                         codegen_fn_attrs.no_sanitize |= SanitizerSet::CFI;
                     } else if item.has_name(sym::kcfi) {
@@ -363,74 +320,62 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
                 }
             }
         } else if attr.has_name(sym::instruction_set) {
-            codegen_fn_attrs.instruction_set = match attr.meta_kind() {
-                Some(MetaItemKind::List(ref items)) => match items.as_slice() {
-                    [NestedMetaItem::MetaItem(set)] => {
-                        let segments =
-                            set.path.segments.iter().map(|x| x.ident.name).collect::<Vec<_>>();
-                        match segments.as_slice() {
-                            [sym::arm, sym::a32] | [sym::arm, sym::t32] => {
-                                if !tcx.sess.target.has_thumb_interworking {
-                                    struct_span_err!(
-                                        tcx.sess.diagnostic(),
-                                        attr.span,
-                                        E0779,
-                                        "target does not support `#[instruction_set]`"
-                                    )
-                                    .emit();
-                                    None
-                                } else if segments[1] == sym::a32 {
-                                    Some(InstructionSetAttr::ArmA32)
-                                } else if segments[1] == sym::t32 {
-                                    Some(InstructionSetAttr::ArmT32)
-                                } else {
-                                    unreachable!()
-                                }
-                            }
-                            _ => {
+            codegen_fn_attrs.instruction_set = attr.meta_item_list().and_then(|l| match &l[..] {
+                [NestedMetaItem::MetaItem(set)] => {
+                    let segments =
+                        set.path.segments.iter().map(|x| x.ident.name).collect::<Vec<_>>();
+                    match segments.as_slice() {
+                        [sym::arm, sym::a32] | [sym::arm, sym::t32] => {
+                            if !tcx.sess.target.has_thumb_interworking {
                                 struct_span_err!(
                                     tcx.sess.diagnostic(),
                                     attr.span,
                                     E0779,
-                                    "invalid instruction set specified",
+                                    "target does not support `#[instruction_set]`"
                                 )
                                 .emit();
                                 None
+                            } else if segments[1] == sym::a32 {
+                                Some(InstructionSetAttr::ArmA32)
+                            } else if segments[1] == sym::t32 {
+                                Some(InstructionSetAttr::ArmT32)
+                            } else {
+                                unreachable!()
                             }
                         }
+                        _ => {
+                            struct_span_err!(
+                                tcx.sess.diagnostic(),
+                                attr.span,
+                                E0779,
+                                "invalid instruction set specified",
+                            )
+                            .emit();
+                            None
+                        }
                     }
-                    [] => {
-                        struct_span_err!(
-                            tcx.sess.diagnostic(),
-                            attr.span,
-                            E0778,
-                            "`#[instruction_set]` requires an argument"
-                        )
-                        .emit();
-                        None
-                    }
-                    _ => {
-                        struct_span_err!(
-                            tcx.sess.diagnostic(),
-                            attr.span,
-                            E0779,
-                            "cannot specify more than one instruction set"
-                        )
-                        .emit();
-                        None
-                    }
-                },
-                _ => {
+                }
+                [] => {
                     struct_span_err!(
                         tcx.sess.diagnostic(),
                         attr.span,
                         E0778,
-                        "must specify an instruction set"
+                        "`#[instruction_set]` requires an argument"
                     )
                     .emit();
                     None
                 }
-            };
+                _ => {
+                    struct_span_err!(
+                        tcx.sess.diagnostic(),
+                        attr.span,
+                        E0779,
+                        "cannot specify more than one instruction set"
+                    )
+                    .emit();
+                    None
+                }
+            })
         } else if attr.has_name(sym::repr) {
             codegen_fn_attrs.alignment = match attr.meta_item_list() {
                 Some(items) => match items.as_slice() {
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 1599ccbb2..f2469fde3 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -181,16 +181,24 @@ fn push_debuginfo_type_name<'tcx>(
                 push_debuginfo_type_name(tcx, inner_type, true, output, visited);
                 match len.kind() {
                     ty::ConstKind::Param(param) => write!(output, ",{}>", param.name).unwrap(),
-                    _ => write!(output, ",{}>", len.eval_usize(tcx, ty::ParamEnv::reveal_all()))
-                        .unwrap(),
+                    _ => write!(
+                        output,
+                        ",{}>",
+                        len.eval_target_usize(tcx, ty::ParamEnv::reveal_all())
+                    )
+                    .unwrap(),
                 }
             } else {
                 output.push('[');
                 push_debuginfo_type_name(tcx, inner_type, true, output, visited);
                 match len.kind() {
                     ty::ConstKind::Param(param) => write!(output, "; {}]", param.name).unwrap(),
-                    _ => write!(output, "; {}]", len.eval_usize(tcx, ty::ParamEnv::reveal_all()))
-                        .unwrap(),
+                    _ => write!(
+                        output,
+                        "; {}]",
+                        len.eval_target_usize(tcx, ty::ParamEnv::reveal_all())
+                    )
+                    .unwrap(),
                 }
             }
         }
@@ -414,6 +422,7 @@ fn push_debuginfo_type_name<'tcx>(
         | ty::Placeholder(..)
         | ty::Alias(..)
         | ty::Bound(..)
+        | ty::GeneratorWitnessMIR(..)
         | ty::GeneratorWitness(..) => {
             bug!(
                 "debuginfo: Trying to create type name for \
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index d81252653..6dea7496f 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -1,8 +1,9 @@
 //! Errors emitted by codegen_ssa
 
 use crate::back::command::Command;
+use crate::fluent_generated as fluent;
 use rustc_errors::{
-    fluent, DiagnosticArgValue, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
+    DiagnosticArgValue, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
     IntoDiagnosticArg,
 };
 use rustc_macros::Diagnostic;
@@ -388,7 +389,7 @@ pub struct LinkerNotFound {
 #[derive(Diagnostic)]
 #[diag(codegen_ssa_unable_to_exe_linker)]
 #[note]
-#[note(command_note)]
+#[note(codegen_ssa_command_note)]
 pub struct UnableToExeLinker {
     pub linker_path: PathBuf,
     pub error: Error,
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 0e6596d4b..ebe9e50ff 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -13,7 +13,7 @@
 
 //! This crate contains codegen code that is used by all codegen backends (LLVM and others).
 //! The backend-agnostic functions of this crate use functions defined in various traits that
-//! have to be implemented by each backends.
+//! have to be implemented by each backend.
 
 #[macro_use]
 extern crate rustc_macros;
@@ -25,7 +25,9 @@ extern crate rustc_middle;
 use rustc_ast as ast;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lrc;
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir::def_id::CrateNum;
+use rustc_macros::fluent_messages;
 use rustc_middle::dep_graph::WorkProduct;
 use rustc_middle::middle::dependency_format::Dependencies;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
@@ -54,6 +56,8 @@ pub mod mono_item;
 pub mod target_features;
 pub mod traits;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 pub struct ModuleCodegen<M> {
     /// The name of the module. When the crate may be saved between
     /// compilations, incremental compilation requires that name be
@@ -159,6 +163,7 @@ pub struct CrateInfo {
     pub dependency_formats: Lrc<Dependencies>,
     pub windows_subsystem: Option<String>,
     pub natvis_debugger_visualizers: BTreeSet<DebuggerVisualizerFile>,
+    pub feature_packed_bundled_libs: bool, // unstable feature flag.
 }
 
 #[derive(Encodable, Decodable)]
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index dd1ac2c74..95aad10fd 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -36,7 +36,7 @@ pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
     // Arguments get assigned to by means of the function being called
     for arg in mir.args_iter() {
-        analyzer.assign(arg, mir::START_BLOCK.start_location());
+        analyzer.assign(arg, DefLocation::Argument);
     }
 
     // If there exists a local definition that dominates all uses of that local,
@@ -64,7 +64,22 @@ enum LocalKind {
     /// A scalar or a scalar pair local that is neither defined nor used.
     Unused,
     /// A scalar or a scalar pair local with a single definition that dominates all uses.
-    SSA(mir::Location),
+    SSA(DefLocation),
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+enum DefLocation {
+    Argument,
+    Body(Location),
+}
+
+impl DefLocation {
+    fn dominates(self, location: Location, dominators: &Dominators<mir::BasicBlock>) -> bool {
+        match self {
+            DefLocation::Argument => true,
+            DefLocation::Body(def) => def.successor_within_block().dominates(location, dominators),
+        }
+    }
 }
 
 struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
@@ -74,17 +89,13 @@ struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
 }
 
 impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> {
-    fn assign(&mut self, local: mir::Local, location: Location) {
+    fn assign(&mut self, local: mir::Local, location: DefLocation) {
         let kind = &mut self.locals[local];
         match *kind {
             LocalKind::ZST => {}
             LocalKind::Memory => {}
-            LocalKind::Unused => {
-                *kind = LocalKind::SSA(location);
-            }
-            LocalKind::SSA(_) => {
-                *kind = LocalKind::Memory;
-            }
+            LocalKind::Unused => *kind = LocalKind::SSA(location),
+            LocalKind::SSA(_) => *kind = LocalKind::Memory,
         }
     }
 
@@ -166,7 +177,7 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
         debug!("visit_assign(place={:?}, rvalue={:?})", place, rvalue);
 
         if let Some(local) = place.as_local() {
-            self.assign(local, location);
+            self.assign(local, DefLocation::Body(location));
             if self.locals[local] != LocalKind::Memory {
                 let decl_span = self.fx.mir.local_decls[local].source_info.span;
                 if !self.fx.rvalue_creates_operand(rvalue, decl_span) {
@@ -189,7 +200,7 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
         match context {
             PlaceContext::MutatingUse(MutatingUseContext::Call)
             | PlaceContext::MutatingUse(MutatingUseContext::Yield) => {
-                self.assign(local, location);
+                self.assign(local, DefLocation::Body(location));
             }
 
             PlaceContext::NonUse(_) | PlaceContext::MutatingUse(MutatingUseContext::Retag) => {}
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 978aff511..57a19a4ab 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -14,9 +14,9 @@ use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::lang_items::LangItem;
 use rustc_index::vec::Idx;
 use rustc_middle::mir::{self, AssertKind, SwitchTargets};
-use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
-use rustc_middle::ty::{self, Instance, Ty, TypeVisitable};
+use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt};
 use rustc_session::config::OptLevel;
 use rustc_span::source_map::Span;
 use rustc_span::{sym, Symbol};
@@ -39,7 +39,6 @@ enum MergingSucc {
 struct TerminatorCodegenHelper<'tcx> {
     bb: mir::BasicBlock,
     terminator: &'tcx mir::Terminator<'tcx>,
-    funclet_bb: Option<mir::BasicBlock>,
 }
 
 impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
@@ -49,28 +48,24 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         &self,
         fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
     ) -> Option<&'b Bx::Funclet> {
-        let funclet_bb = self.funclet_bb?;
-        if base::wants_msvc_seh(fx.cx.tcx().sess) {
-            // If `landing_pad_for` hasn't been called yet to create the `Funclet`,
-            // it has to be now. This may not seem necessary, as RPO should lead
-            // to all the unwind edges being visited (and so to `landing_pad_for`
-            // getting called for them), before building any of the blocks inside
-            // the funclet itself - however, if MIR contains edges that end up not
-            // being needed in the LLVM IR after monomorphization, the funclet may
-            // be unreachable, and we don't have yet a way to skip building it in
-            // such an eventuality (which may be a better solution than this).
-            if fx.funclets[funclet_bb].is_none() {
-                fx.landing_pad_for(funclet_bb);
-            }
-
-            Some(
-                fx.funclets[funclet_bb]
-                    .as_ref()
-                    .expect("landing_pad_for didn't also create funclets entry"),
-            )
-        } else {
-            None
+        let cleanup_kinds = (&fx.cleanup_kinds).as_ref()?;
+        let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?;
+        // If `landing_pad_for` hasn't been called yet to create the `Funclet`,
+        // it has to be now. This may not seem necessary, as RPO should lead
+        // to all the unwind edges being visited (and so to `landing_pad_for`
+        // getting called for them), before building any of the blocks inside
+        // the funclet itself - however, if MIR contains edges that end up not
+        // being needed in the LLVM IR after monomorphization, the funclet may
+        // be unreachable, and we don't have yet a way to skip building it in
+        // such an eventuality (which may be a better solution than this).
+        if fx.funclets[funclet_bb].is_none() {
+            fx.landing_pad_for(funclet_bb);
         }
+        Some(
+            fx.funclets[funclet_bb]
+                .as_ref()
+                .expect("landing_pad_for didn't also create funclets entry"),
+        )
     }
 
     /// Get a basic block (creating it if necessary), possibly with cleanup
@@ -104,23 +99,24 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         fx: &mut FunctionCx<'a, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> (bool, bool) {
-        let target_funclet = fx.cleanup_kinds[target].funclet_bb(target);
-        let (needs_landing_pad, is_cleanupret) = match (self.funclet_bb, target_funclet) {
-            (None, None) => (false, false),
-            (None, Some(_)) => (true, false),
-            (Some(_), None) => {
-                let span = self.terminator.source_info.span;
-                span_bug!(span, "{:?} - jump out of cleanup?", self.terminator);
-            }
-            (Some(f), Some(t_f)) => {
-                if f == t_f || !base::wants_msvc_seh(fx.cx.tcx().sess) {
-                    (false, false)
-                } else {
-                    (true, true)
+        if let Some(ref cleanup_kinds) = fx.cleanup_kinds {
+            let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb);
+            let target_funclet = cleanup_kinds[target].funclet_bb(target);
+            let (needs_landing_pad, is_cleanupret) = match (funclet_bb, target_funclet) {
+                (None, None) => (false, false),
+                (None, Some(_)) => (true, false),
+                (Some(f), Some(t_f)) => (f != t_f, f != t_f),
+                (Some(_), None) => {
+                    let span = self.terminator.source_info.span;
+                    span_bug!(span, "{:?} - jump out of cleanup?", self.terminator);
                 }
-            }
-        };
-        (needs_landing_pad, is_cleanupret)
+            };
+            (needs_landing_pad, is_cleanupret)
+        } else {
+            let needs_landing_pad = !fx.mir[self.bb].is_cleanup && fx.mir[target].is_cleanup;
+            let is_cleanupret = false;
+            (needs_landing_pad, is_cleanupret)
+        }
     }
 
     fn funclet_br<Bx: BuilderMethods<'a, 'tcx>>(
@@ -456,86 +452,84 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             args1 = [place.llval];
             &args1[..]
         };
-        let (drop_fn, fn_abi) = match ty.kind() {
-            // FIXME(eddyb) perhaps move some of this logic into
-            // `Instance::resolve_drop_in_place`?
-            ty::Dynamic(_, _, ty::Dyn) => {
-                // IN THIS ARM, WE HAVE:
-                // ty = *mut (dyn Trait)
-                // which is: exists<T> ( *mut T,    Vtable<T: Trait> )
-                //                       args[0]    args[1]
-                //
-                // args = ( Data, Vtable )
-                //                  |
-                //                  v
-                //                /-------\
-                //                | ...   |
-                //                \-------/
-                //
-                let virtual_drop = Instance {
-                    def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
-                    substs: drop_fn.substs,
-                };
-                debug!("ty = {:?}", ty);
-                debug!("drop_fn = {:?}", drop_fn);
-                debug!("args = {:?}", args);
-                let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
-                let vtable = args[1];
-                // Truncate vtable off of args list
-                args = &args[..1];
-                (
-                    meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
-                        .get_fn(bx, vtable, ty, &fn_abi),
-                    fn_abi,
-                )
-            }
-            ty::Dynamic(_, _, ty::DynStar) => {
-                // IN THIS ARM, WE HAVE:
-                // ty = *mut (dyn* Trait)
-                // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
-                //
-                // args = [ * ]
-                //          |
-                //          v
-                //      ( Data, Vtable )
-                //                |
-                //                v
-                //              /-------\
-                //              | ...   |
-                //              \-------/
-                //
-                //
-                // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
-                //
-                // data = &(*args[0]).0    // gives a pointer to Data above (really the same pointer)
-                // vtable = (*args[0]).1   // loads the vtable out
-                // (data, vtable)          // an equivalent Rust `*mut dyn Trait`
-                //
-                // SO THEN WE CAN USE THE ABOVE CODE.
-                let virtual_drop = Instance {
-                    def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
-                    substs: drop_fn.substs,
-                };
-                debug!("ty = {:?}", ty);
-                debug!("drop_fn = {:?}", drop_fn);
-                debug!("args = {:?}", args);
-                let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
-                let data = args[0];
-                let data_ty = bx.cx().backend_type(place.layout);
-                let vtable_ptr =
-                    bx.gep(data_ty, data, &[bx.cx().const_i32(0), bx.cx().const_i32(1)]);
-                let vtable = bx.load(bx.type_i8p(), vtable_ptr, abi::Align::ONE);
-                // Truncate vtable off of args list
-                args = &args[..1];
-                debug!("args' = {:?}", args);
-                (
-                    meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
-                        .get_fn(bx, vtable, ty, &fn_abi),
-                    fn_abi,
-                )
-            }
-            _ => (bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty())),
-        };
+        let (drop_fn, fn_abi) =
+            match ty.kind() {
+                // FIXME(eddyb) perhaps move some of this logic into
+                // `Instance::resolve_drop_in_place`?
+                ty::Dynamic(_, _, ty::Dyn) => {
+                    // IN THIS ARM, WE HAVE:
+                    // ty = *mut (dyn Trait)
+                    // which is: exists<T> ( *mut T,    Vtable<T: Trait> )
+                    //                       args[0]    args[1]
+                    //
+                    // args = ( Data, Vtable )
+                    //                  |
+                    //                  v
+                    //                /-------\
+                    //                | ...   |
+                    //                \-------/
+                    //
+                    let virtual_drop = Instance {
+                        def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+                        substs: drop_fn.substs,
+                    };
+                    debug!("ty = {:?}", ty);
+                    debug!("drop_fn = {:?}", drop_fn);
+                    debug!("args = {:?}", args);
+                    let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
+                    let vtable = args[1];
+                    // Truncate vtable off of args list
+                    args = &args[..1];
+                    (
+                        meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
+                            .get_fn(bx, vtable, ty, &fn_abi),
+                        fn_abi,
+                    )
+                }
+                ty::Dynamic(_, _, ty::DynStar) => {
+                    // IN THIS ARM, WE HAVE:
+                    // ty = *mut (dyn* Trait)
+                    // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
+                    //
+                    // args = [ * ]
+                    //          |
+                    //          v
+                    //      ( Data, Vtable )
+                    //                |
+                    //                v
+                    //              /-------\
+                    //              | ...   |
+                    //              \-------/
+                    //
+                    //
+                    // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
+                    //
+                    // data = &(*args[0]).0    // gives a pointer to Data above (really the same pointer)
+                    // vtable = (*args[0]).1   // loads the vtable out
+                    // (data, vtable)          // an equivalent Rust `*mut dyn Trait`
+                    //
+                    // SO THEN WE CAN USE THE ABOVE CODE.
+                    let virtual_drop = Instance {
+                        def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+                        substs: drop_fn.substs,
+                    };
+                    debug!("ty = {:?}", ty);
+                    debug!("drop_fn = {:?}", drop_fn);
+                    debug!("args = {:?}", args);
+                    let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
+                    let meta_ptr = place.project_field(bx, 1);
+                    let meta = bx.load_operand(meta_ptr);
+                    // Truncate vtable off of args list
+                    args = &args[..1];
+                    debug!("args' = {:?}", args);
+                    (
+                        meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
+                            .get_fn(bx, meta.immediate(), ty, &fn_abi),
+                        fn_abi,
+                    )
+                }
+                _ => (bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty())),
+            };
         helper.do_call(
             self,
             bx,
@@ -569,11 +563,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // with #[rustc_inherit_overflow_checks] and inlined from
         // another crate (mostly core::num generic/#[inline] fns),
         // while the current crate doesn't use overflow checks.
-        // NOTE: Unlike binops, negation doesn't have its own
-        // checked operation, just a comparison with the minimum
-        // value, so we have to check for the assert message.
-        if !bx.check_overflow() {
-            if let AssertKind::OverflowNeg(_) = *msg {
+        if !bx.cx().check_overflow() {
+            let overflow_not_to_check = match msg {
+                AssertKind::OverflowNeg(..) => true,
+                AssertKind::Overflow(op, ..) => op.is_checkable(),
+                _ => false,
+            };
+            if overflow_not_to_check {
                 const_cond = Some(expected);
             }
         }
@@ -659,35 +655,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // Emit a panic or a no-op for `assert_*` intrinsics.
         // These are intrinsics that compile to panics so that we can get a message
         // which mentions the offending type, even from a const context.
-        #[derive(Debug, PartialEq)]
-        enum AssertIntrinsic {
-            Inhabited,
-            ZeroValid,
-            MemUninitializedValid,
-        }
-        let panic_intrinsic = intrinsic.and_then(|i| match i {
-            sym::assert_inhabited => Some(AssertIntrinsic::Inhabited),
-            sym::assert_zero_valid => Some(AssertIntrinsic::ZeroValid),
-            sym::assert_mem_uninitialized_valid => Some(AssertIntrinsic::MemUninitializedValid),
-            _ => None,
-        });
-        if let Some(intrinsic) = panic_intrinsic {
-            use AssertIntrinsic::*;
-
+        let panic_intrinsic = intrinsic.and_then(|s| ValidityRequirement::from_intrinsic(s));
+        if let Some(requirement) = panic_intrinsic {
             let ty = instance.unwrap().substs.type_at(0);
+
+            let do_panic = !bx
+                .tcx()
+                .check_validity_requirement((requirement, bx.param_env().and(ty)))
+                .expect("expect to have layout during codegen");
+
             let layout = bx.layout_of(ty);
-            let do_panic = match intrinsic {
-                Inhabited => layout.abi.is_uninhabited(),
-                ZeroValid => !bx.tcx().permits_zero_init(layout),
-                MemUninitializedValid => !bx.tcx().permits_uninit_init(layout),
-            };
+
             Some(if do_panic {
                 let msg_str = with_no_visible_paths!({
                     with_no_trimmed_paths!({
                         if layout.abi.is_uninhabited() {
                             // Use this error even for the other intrinsics as it is more precise.
                             format!("attempted to instantiate uninhabited type `{}`", ty)
-                        } else if intrinsic == ZeroValid {
+                        } else if requirement == ValidityRequirement::Zero {
                             format!("attempted to zero-initialize type `{}`, which is invalid", ty)
                         } else {
                             format!(
@@ -781,7 +766,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         };
 
         let extra_args = &args[sig.inputs().skip_binder().len()..];
-        let extra_args = bx.tcx().mk_type_list(extra_args.iter().map(|op_arg| {
+        let extra_args = bx.tcx().mk_type_list_from_iter(extra_args.iter().map(|op_arg| {
             let op_ty = op_arg.ty(self.mir, bx.tcx());
             self.monomorphize(op_ty)
         }));
@@ -1253,9 +1238,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) -> MergingSucc {
         debug!("codegen_terminator: {:?}", terminator);
 
-        // Create the cleanup bundle, if needed.
-        let funclet_bb = self.cleanup_kinds[bb].funclet_bb(bb);
-        let helper = TerminatorCodegenHelper { bb, terminator, funclet_bb };
+        let helper = TerminatorCodegenHelper { bb, terminator };
 
         let mergeable_succ = || {
             // Note: any call to `switch_to_block` will invalidate a `true` value
@@ -1547,7 +1530,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             slot
         } else {
             let layout = cx.layout_of(
-                cx.tcx().intern_tup(&[cx.tcx().mk_mut_ptr(cx.tcx().types.u8), cx.tcx().types.i32]),
+                cx.tcx().mk_tup(&[cx.tcx().mk_mut_ptr(cx.tcx().types.u8), cx.tcx().types.i32]),
             );
             let slot = PlaceRef::alloca(bx, layout);
             self.personality_slot = Some(slot);
@@ -1801,8 +1784,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         match (src.layout.abi, dst.layout.abi) {
             (abi::Abi::Scalar(src_scalar), abi::Abi::Scalar(dst_scalar)) => {
                 // HACK(eddyb) LLVM doesn't like `bitcast`s between pointers and non-pointers.
-                let src_is_ptr = src_scalar.primitive() == abi::Pointer;
-                let dst_is_ptr = dst_scalar.primitive() == abi::Pointer;
+                let src_is_ptr = matches!(src_scalar.primitive(), abi::Pointer(_));
+                let dst_is_ptr = matches!(dst_scalar.primitive(), abi::Pointer(_));
                 if src_is_ptr == dst_is_ptr {
                     assert_eq!(src.layout.size, dst.layout.size);
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
index e9bc40c33..708f3bc0c 100644
--- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -385,10 +385,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     calculate_debuginfo_offset(bx, local, &var, base);
 
                 // Create a variable which will be a pointer to the actual value
-                let ptr_ty = bx.tcx().mk_ty(ty::RawPtr(ty::TypeAndMut {
-                    mutbl: mir::Mutability::Mut,
-                    ty: place.layout.ty,
-                }));
+                let ptr_ty = bx
+                    .tcx()
+                    .mk_ptr(ty::TypeAndMut { mutbl: mir::Mutability::Mut, ty: place.layout.ty });
                 let ptr_layout = bx.layout_of(ptr_ty);
                 let alloca = PlaceRef::alloca(bx, ptr_layout);
                 bx.set_var_name(alloca.llval, &(var.name.to_string() + ".dbg.spill"));
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 766dc74cb..7af7fc92d 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -218,9 +218,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 args[1].val.unaligned_volatile_store(bx, dst);
                 return;
             }
-            sym::add_with_overflow
-            | sym::sub_with_overflow
-            | sym::mul_with_overflow
             | sym::unchecked_div
             | sym::unchecked_rem
             | sym::unchecked_shl
@@ -232,28 +229,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let ty = arg_tys[0];
                 match int_type_width_signed(ty, bx.tcx()) {
                     Some((_width, signed)) => match name {
-                        sym::add_with_overflow
-                        | sym::sub_with_overflow
-                        | sym::mul_with_overflow => {
-                            let op = match name {
-                                sym::add_with_overflow => OverflowOp::Add,
-                                sym::sub_with_overflow => OverflowOp::Sub,
-                                sym::mul_with_overflow => OverflowOp::Mul,
-                                _ => bug!(),
-                            };
-                            let (val, overflow) =
-                                bx.checked_binop(op, ty, args[0].immediate(), args[1].immediate());
-                            // Convert `i1` to a `bool`, and write it to the out parameter
-                            let val = bx.from_immediate(val);
-                            let overflow = bx.from_immediate(overflow);
-
-                            let dest = result.project_field(bx, 0);
-                            bx.store(val, dest.llval, dest.align);
-                            let dest = result.project_field(bx, 1);
-                            bx.store(overflow, dest.llval, dest.align);
-
-                            return;
-                        }
                         sym::exact_div => {
                             if signed {
                                 bx.exactsdiv(args[0].immediate(), args[1].immediate())
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index 79c66a955..2ec9fdbf4 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -1,8 +1,9 @@
+use crate::base;
 use crate::traits::*;
 use rustc_middle::mir;
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout};
-use rustc_middle::ty::{self, Instance, Ty, TypeFoldable, TypeVisitable};
+use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_target::abi::call::{FnAbi, PassMode};
 
 use std::iter;
@@ -58,7 +59,7 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
     cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>>,
 
     /// The funclet status of each basic block
-    cleanup_kinds: IndexVec<mir::BasicBlock, analyze::CleanupKind>,
+    cleanup_kinds: Option<IndexVec<mir::BasicBlock, analyze::CleanupKind>>,
 
     /// When targeting MSVC, this stores the cleanup info for each funclet BB.
     /// This is initialized at the same time as the `landing_pads` entry for the
@@ -104,7 +105,7 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     pub fn monomorphize<T>(&self, value: T) -> T
     where
-        T: Copy + TypeFoldable<'tcx>,
+        T: Copy + TypeFoldable<TyCtxt<'tcx>>,
     {
         debug!("monomorphize: self.instance={:?}", self.instance);
         self.instance.subst_mir_and_normalize_erasing_regions(
@@ -166,7 +167,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         start_bx.set_personality_fn(cx.eh_personality());
     }
 
-    let cleanup_kinds = analyze::cleanup_kinds(&mir);
+    let cleanup_kinds = base::wants_msvc_seh(cx.tcx().sess).then(|| analyze::cleanup_kinds(&mir));
+
     let cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>> =
         mir.basic_blocks
             .indices()
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index fbe30154a..cf02f59f6 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -9,7 +9,7 @@ use rustc_middle::mir;
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Ty};
-use rustc_target::abi::{Abi, Align, FieldsShape, Int, TagEncoding};
+use rustc_target::abi::{Abi, Align, FieldsShape, Int, Pointer, TagEncoding};
 use rustc_target::abi::{VariantIdx, Variants};
 
 #[derive(Copy, Clone, Debug)]
@@ -209,6 +209,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         bx: &mut Bx,
         cast_to: Ty<'tcx>,
     ) -> V {
+        let dl = &bx.tcx().data_layout;
         let cast_to_layout = bx.cx().layout_of(cast_to);
         let cast_to_size = cast_to_layout.layout.size();
         let cast_to = bx.cx().immediate_backend_type(cast_to_layout);
@@ -250,12 +251,14 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
             TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
                 // Cast to an integer so we don't have to treat a pointer as a
                 // special case.
-                let (tag, tag_llty) = if tag_scalar.primitive().is_ptr() {
-                    let t = bx.type_isize();
-                    let tag = bx.ptrtoint(tag_imm, t);
-                    (tag, t)
-                } else {
-                    (tag_imm, bx.cx().immediate_backend_type(tag_op.layout))
+                let (tag, tag_llty) = match tag_scalar.primitive() {
+                    // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+                    Pointer(_) => {
+                        let t = bx.type_from_integer(dl.ptr_sized_integer());
+                        let tag = bx.ptrtoint(tag_imm, t);
+                        (tag, t)
+                    }
+                    _ => (tag_imm, bx.cx().immediate_backend_type(tag_op.layout)),
                 };
 
                 let tag_size = tag_scalar.size(bx.cx());
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 23196c8cb..3d856986f 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -13,6 +13,7 @@ use rustc_middle::ty::cast::{CastTy, IntTy};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt};
 use rustc_span::source_map::{Span, DUMMY_SP};
+use rustc_target::abi::VariantIdx;
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     #[instrument(level = "trace", skip(self, bx))]
@@ -99,24 +100,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     }
                 }
 
-                let count =
-                    self.monomorphize(count).eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
+                let count = self
+                    .monomorphize(count)
+                    .eval_target_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
 
                 bx.write_operand_repeatedly(cg_elem, count, dest);
             }
 
             mir::Rvalue::Aggregate(ref kind, ref operands) => {
-                let (dest, active_field_index) = match **kind {
-                    mir::AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => {
-                        dest.codegen_set_discr(bx, variant_index);
-                        if bx.tcx().adt_def(adt_did).is_enum() {
-                            (dest.project_downcast(bx, variant_index), active_field_index)
-                        } else {
-                            (dest, active_field_index)
-                        }
+                let (variant_index, variant_dest, active_field_index) = match **kind {
+                    mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {
+                        let variant_dest = dest.project_downcast(bx, variant_index);
+                        (variant_index, variant_dest, active_field_index)
                     }
-                    _ => (dest, None),
+                    _ => (VariantIdx::from_u32(0), dest, None),
                 };
+                if active_field_index.is_some() {
+                    assert_eq!(operands.len(), 1);
+                }
                 for (i, operand) in operands.iter().enumerate() {
                     let op = self.codegen_operand(bx, operand);
                     // Do not generate stores and GEPis for zero-sized fields.
@@ -124,13 +125,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         let field_index = active_field_index.unwrap_or(i);
                         let field = if let mir::AggregateKind::Array(_) = **kind {
                             let llindex = bx.cx().const_usize(field_index as u64);
-                            dest.project_index(bx, llindex)
+                            variant_dest.project_index(bx, llindex)
                         } else {
-                            dest.project_field(bx, field_index)
+                            variant_dest.project_field(bx, field_index)
                         };
                         op.val.store(bx, field);
                     }
                 }
+                dest.codegen_set_discr(bx, variant_index);
             }
 
             _ => {
@@ -411,7 +413,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     lhs.layout.ty,
                 );
                 let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty);
-                let operand_ty = bx.tcx().intern_tup(&[val_ty, bx.tcx().types.bool]);
+                let operand_ty = bx.tcx().mk_tup(&[val_ty, bx.tcx().types.bool]);
                 OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) }
             }
 
@@ -491,7 +493,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         if let Some(index) = place.as_local() {
             if let LocalRef::Operand(Some(op)) = self.locals[index] {
                 if let ty::Array(_, n) = op.layout.ty.kind() {
-                    let n = n.eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
+                    let n = n.eval_target_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
                     return bx.cx().const_usize(n);
                 }
             }
@@ -650,15 +652,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         rhs: Bx::Value,
         input_ty: Ty<'tcx>,
     ) -> OperandValue<Bx::Value> {
-        // This case can currently arise only from functions marked
-        // with #[rustc_inherit_overflow_checks] and inlined from
-        // another crate (mostly core::num generic/#[inline] fns),
-        // while the current crate doesn't use overflow checks.
-        if !bx.cx().check_overflow() {
-            let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty);
-            return OperandValue::Pair(val, bx.cx().const_bool(false));
-        }
-
         let (val, of) = match op {
             // These are checked using intrinsics
             mir::BinOp::Add | mir::BinOp::Sub | mir::BinOp::Mul => {
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
index 19452c8cd..60fbceb34 100644
--- a/compiler/rustc_codegen_ssa/src/mir/statement.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -91,6 +91,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::StatementKind::FakeRead(..)
             | mir::StatementKind::Retag { .. }
             | mir::StatementKind::AscribeUserType(..)
+            | mir::StatementKind::ConstEvalCounter
             | mir::StatementKind::Nop => {}
         }
     }
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index 739963fff..e59fad99a 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -3,12 +3,12 @@ use rustc_attr::InstructionSetAttr;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::Applicability;
-use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{DefIdTree, TyCtxt};
 use rustc_session::parse::feature_err;
 use rustc_session::Session;
 use rustc_span::symbol::sym;
@@ -185,7 +185,7 @@ const X86_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("avx512vpopcntdq", Some(sym::avx512_target_feature)),
     ("bmi1", None),
     ("bmi2", None),
-    ("cmpxchg16b", Some(sym::cmpxchg16b_target_feature)),
+    ("cmpxchg16b", None),
     ("ermsb", Some(sym::ermsb_target_feature)),
     ("f16c", None),
     ("fma", None),
@@ -286,6 +286,7 @@ const WASM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("mutable-globals", Some(sym::wasm_target_feature)),
     ("nontrapping-fptoint", Some(sym::wasm_target_feature)),
     ("reference-types", Some(sym::wasm_target_feature)),
+    ("relaxed-simd", Some(sym::wasm_target_feature)),
     ("sign-ext", Some(sym::wasm_target_feature)),
     ("simd128", None),
     // tidy-alphabetical-end
@@ -393,7 +394,6 @@ pub fn from_target_feature(
                 Some(sym::sse4a_target_feature) => rust_features.sse4a_target_feature,
                 Some(sym::tbm_target_feature) => rust_features.tbm_target_feature,
                 Some(sym::wasm_target_feature) => rust_features.wasm_target_feature,
-                Some(sym::cmpxchg16b_target_feature) => rust_features.cmpxchg16b_target_feature,
                 Some(sym::movbe_target_feature) => rust_features.movbe_target_feature,
                 Some(sym::rtm_target_feature) => rust_features.rtm_target_feature,
                 Some(sym::ermsb_target_feature) => rust_features.ermsb_target_feature,
@@ -440,12 +440,9 @@ fn asm_target_features(tcx: TyCtxt<'_>, did: DefId) -> &FxHashSet<Symbol> {
 /// Checks the function annotated with `#[target_feature]` is not a safe
 /// trait method implementation, reporting an error if it is.
 pub fn check_target_feature_trait_unsafe(tcx: TyCtxt<'_>, id: LocalDefId, attr_span: Span) {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(id);
-    let node = tcx.hir().get(hir_id);
-    if let hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }) = node {
-        let parent_id = tcx.hir().get_parent_item(hir_id);
-        let parent_item = tcx.hir().expect_item(parent_id.def_id);
-        if let hir::ItemKind::Impl(hir::Impl { of_trait: Some(_), .. }) = parent_item.kind {
+    if let DefKind::AssocFn = tcx.def_kind(id) {
+        let parent_id = tcx.local_parent(id);
+        if let DefKind::Impl { of_trait: true } = tcx.def_kind(parent_id) {
             tcx.sess
                 .struct_span_err(
                     attr_span,
diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs
index 5c35070ea..64bebe50d 100644
--- a/compiler/rustc_codegen_ssa/src/traits/backend.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs
@@ -57,6 +57,10 @@ impl<'tcx, T> Backend<'tcx> for T where
 }
 
 pub trait CodegenBackend {
+    /// Locale resources for diagnostic messages - a string the content of the Fluent resource.
+    /// Called before `init` so that all other functions are able to emit translatable diagnostics.
+    fn locale_resource(&self) -> &'static str;
+
     fn init(&self, _sess: &Session) {}
     fn print(&self, _req: PrintRequest, _sess: &Session) {}
     fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec<Symbol> {
diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs
index e0e8ffa89..9826256a4 100644
--- a/compiler/rustc_codegen_ssa/src/traits/write.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/write.rs
@@ -8,6 +8,7 @@ use rustc_middle::dep_graph::WorkProduct;
 pub trait WriteBackendMethods: 'static + Sized + Clone {
     type Module: Send + Sync;
     type TargetMachine;
+    type TargetMachineError;
     type ModuleBuffer: ModuleBufferMethods;
     type ThinData: Send + Sync;
     type ThinBuffer: ThinBufferMethods;