Merging debian version 1.65.0+dfsg1-2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

29 files changed, 1027 insertions, 470 deletions

diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml
index f9a5463ef..74115353a 100644
--- a/compiler/rustc_codegen_llvm/Cargo.toml
+++ b/compiler/rustc_codegen_llvm/Cargo.toml
@@ -13,6 +13,7 @@ cstr = "0.2"
 libc = "0.2"
 libloading = "0.7.1"
 measureme = "10.0.0"
+object = { version = "0.29.0", default-features = false, features = ["std", "read_core", "archive", "coff", "elf", "macho", "pe"] }
 tracing = "0.1"
 rustc_middle = { path = "../rustc_middle" }
 rustc-demangle = "0.1.21"
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index 9eb3574e7..26f5225f6 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -19,6 +19,7 @@ use rustc_target::abi::call::ArgAbi;
 pub use rustc_target::abi::call::*;
 use rustc_target::abi::{self, HasDataLayout, Int};
 pub use rustc_target::spec::abi::Abi;
+use rustc_target::spec::SanitizerSet;
 
 use libc::c_uint;
 use smallvec::SmallVec;
@@ -90,6 +91,13 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'
         if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
             attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx));
         }
+    } else if cx.tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) {
+        // If we're not optimising, *but* memory sanitizer is on, emit noundef, since it affects
+        // memory sanitizer's behavior.
+
+        if regular.contains(ArgAttribute::NoUndef) {
+            attrs.push(llvm::AttributeKind::NoUndef.create_attr(cx.llcx));
+        }
     }
 
     attrs
@@ -213,7 +221,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst)
         } else if self.is_unsized_indirect() {
             bug!("unsized `ArgAbi` must be handled through `store_fn_arg`");
-        } else if let PassMode::Cast(cast) = self.mode {
+        } else if let PassMode::Cast(cast, _) = &self.mode {
             // FIXME(eddyb): Figure out when the simpler Store is safe, clang
             // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}.
             let can_store_through_cast_ptr = false;
@@ -283,7 +291,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             }
             PassMode::Direct(_)
             | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ }
-            | PassMode::Cast(_) => {
+            | PassMode::Cast(..) => {
                 let next_arg = next();
                 self.store(bx, next_arg, dst);
             }
@@ -325,20 +333,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
     fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
         // Ignore "extra" args from the call site for C variadic functions.
         // Only the "fixed" args are part of the LLVM function signature.
-        let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args };
+        let args =
+            if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args };
 
-        let args_capacity: usize = args.iter().map(|arg|
-            if arg.pad.is_some() { 1 } else { 0 } +
-            if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 }
-        ).sum();
+        // This capacity calculation is approximate.
         let mut llargument_tys = Vec::with_capacity(
-            if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity,
+            self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 },
         );
 
-        let llreturn_ty = match self.ret.mode {
+        let llreturn_ty = match &self.ret.mode {
             PassMode::Ignore => cx.type_void(),
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
-            PassMode::Cast(cast) => cast.llvm_type(cx),
+            PassMode::Cast(cast, _) => cast.llvm_type(cx),
             PassMode::Indirect { .. } => {
                 llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx)));
                 cx.type_void()
@@ -346,12 +352,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         };
 
         for arg in args {
-            // add padding
-            if let Some(ty) = arg.pad {
-                llargument_tys.push(ty.llvm_type(cx));
-            }
-
-            let llarg_ty = match arg.mode {
+            let llarg_ty = match &arg.mode {
                 PassMode::Ignore => continue,
                 PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx),
                 PassMode::Pair(..) => {
@@ -366,7 +367,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true));
                     continue;
                 }
-                PassMode::Cast(cast) => cast.llvm_type(cx),
+                PassMode::Cast(cast, pad_i32) => {
+                    // add padding
+                    if *pad_i32 {
+                        llargument_tys.push(Reg::i32().llvm_type(cx));
+                    }
+                    cast.llvm_type(cx)
+                }
                 PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
                     cx.type_ptr_to(arg.memory_ty(cx))
                 }
@@ -426,46 +433,46 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             i += 1;
             i - 1
         };
-        match self.ret.mode {
-            PassMode::Direct(ref attrs) => {
+        match &self.ret.mode {
+            PassMode::Direct(attrs) => {
                 attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
             }
-            PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
+            PassMode::Indirect { attrs, extra_attrs: _, on_stack } => {
                 assert!(!on_stack);
                 let i = apply(attrs);
                 let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx));
                 attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]);
             }
-            PassMode::Cast(cast) => {
+            PassMode::Cast(cast, _) => {
                 cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
             }
             _ => {}
         }
-        for arg in &self.args {
-            if arg.pad.is_some() {
-                apply(&ArgAttributes::new());
-            }
-            match arg.mode {
+        for arg in self.args.iter() {
+            match &arg.mode {
                 PassMode::Ignore => {}
-                PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => {
+                PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => {
                     let i = apply(attrs);
                     let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx));
                     attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]);
                 }
-                PassMode::Direct(ref attrs)
-                | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => {
+                PassMode::Direct(attrs)
+                | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => {
                     apply(attrs);
                 }
-                PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => {
+                PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => {
                     assert!(!on_stack);
                     apply(attrs);
                     apply(extra_attrs);
                 }
-                PassMode::Pair(ref a, ref b) => {
+                PassMode::Pair(a, b) => {
                     apply(a);
                     apply(b);
                 }
-                PassMode::Cast(cast) => {
+                PassMode::Cast(cast, pad_i32) => {
+                    if *pad_i32 {
+                        apply(&ArgAttributes::new());
+                    }
                     apply(&cast.attrs);
                 }
             }
@@ -488,17 +495,17 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             i += 1;
             i - 1
         };
-        match self.ret.mode {
-            PassMode::Direct(ref attrs) => {
+        match &self.ret.mode {
+            PassMode::Direct(attrs) => {
                 attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite);
             }
-            PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
+            PassMode::Indirect { attrs, extra_attrs: _, on_stack } => {
                 assert!(!on_stack);
                 let i = apply(bx.cx, attrs);
                 let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx));
                 attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]);
             }
-            PassMode::Cast(cast) => {
+            PassMode::Cast(cast, _) => {
                 cast.attrs.apply_attrs_to_callsite(
                     llvm::AttributePlace::ReturnValue,
                     &bx.cx,
@@ -517,13 +524,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 }
             }
         }
-        for arg in &self.args {
-            if arg.pad.is_some() {
-                apply(bx.cx, &ArgAttributes::new());
-            }
-            match arg.mode {
+        for arg in self.args.iter() {
+            match &arg.mode {
                 PassMode::Ignore => {}
-                PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => {
+                PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => {
                     let i = apply(bx.cx, attrs);
                     let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx));
                     attributes::apply_to_callsite(
@@ -532,23 +536,22 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         &[byval],
                     );
                 }
-                PassMode::Direct(ref attrs)
-                | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => {
+                PassMode::Direct(attrs)
+                | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => {
                     apply(bx.cx, attrs);
                 }
-                PassMode::Indirect {
-                    ref attrs,
-                    extra_attrs: Some(ref extra_attrs),
-                    on_stack: _,
-                } => {
+                PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack: _ } => {
                     apply(bx.cx, attrs);
                     apply(bx.cx, extra_attrs);
                 }
-                PassMode::Pair(ref a, ref b) => {
+                PassMode::Pair(a, b) => {
                     apply(bx.cx, a);
                     apply(bx.cx, b);
                 }
-                PassMode::Cast(cast) => {
+                PassMode::Cast(cast, pad_i32) => {
+                    if *pad_i32 {
+                        apply(bx.cx, &ArgAttributes::new());
+                    }
                     apply(bx.cx, &cast.attrs);
                 }
             }
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index a53946995..5202ac697 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -3,7 +3,6 @@ use crate::builder::Builder;
 use crate::common::Funclet;
 use crate::context::CodegenCx;
 use crate::llvm;
-use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -20,7 +19,6 @@ use rustc_target::asm::*;
 
 use libc::{c_char, c_uint};
 use smallvec::SmallVec;
-use tracing::debug;
 
 impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     fn codegen_inline_asm(
@@ -419,13 +417,6 @@ pub(crate) fn inline_asm_call<'ll>(
         let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len());
         debug!("constraint verification result: {:?}", constraints_ok);
         if constraints_ok {
-            if unwind && llvm_util::get_version() < (13, 0, 0) {
-                bx.cx.sess().span_fatal(
-                    line_spans[0],
-                    "unwinding from inline assembly is only supported on llvm >= 13.",
-                );
-            }
-
             let v = llvm::LLVMRustInlineAsm(
                 fty,
                 asm.as_ptr().cast(),
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index aabbe8ac2..eff2436d4 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -35,6 +35,10 @@ pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attrib
 /// Get LLVM attribute for the provided inline heuristic.
 #[inline]
 fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> {
+    if !cx.tcx.sess.opts.unstable_opts.inline_llvm {
+        // disable LLVM inlining
+        return Some(AttributeKind::NoInline.create_attr(cx.llcx));
+    }
     match inline {
         InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
         InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)),
@@ -386,7 +390,8 @@ pub fn from_fn_attrs<'ll, 'tcx>(
     ) {
         let span = cx
             .tcx
-            .get_attr(instance.def_id(), sym::target_feature)
+            .get_attrs(instance.def_id(), sym::target_feature)
+            .next()
             .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span);
         let msg = format!(
             "the target features {} must all be either enabled or disabled together",
diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs
index 27039cda2..ed96355a0 100644
--- a/compiler/rustc_codegen_llvm/src/back/archive.rs
+++ b/compiler/rustc_codegen_llvm/src/back/archive.rs
@@ -8,10 +8,11 @@ use std::path::{Path, PathBuf};
 use std::ptr;
 use std::str;
 
+use crate::common;
 use crate::llvm::archive_ro::{ArchiveRO, Child};
 use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport};
 use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
-use rustc_session::cstore::{DllCallingConvention, DllImport};
+use rustc_session::cstore::DllImport;
 use rustc_session::Session;
 
 /// Helper for adding many files to an archive.
@@ -103,29 +104,28 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
         lib_name: &str,
         dll_imports: &[DllImport],
         tmpdir: &Path,
+        is_direct_dependency: bool,
     ) -> PathBuf {
+        let name_suffix = if is_direct_dependency { "_imports" } else { "_imports_indirect" };
         let output_path = {
             let mut output_path: PathBuf = tmpdir.to_path_buf();
-            output_path.push(format!("{}_imports", lib_name));
+            output_path.push(format!("{}{}", lib_name, name_suffix));
             output_path.with_extension("lib")
         };
 
         let target = &sess.target;
-        let mingw_gnu_toolchain = target.vendor == "pc"
-            && target.os == "windows"
-            && target.env == "gnu"
-            && target.abi.is_empty();
+        let mingw_gnu_toolchain = common::is_mingw_gnu_toolchain(target);
 
         let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports
             .iter()
             .map(|import: &DllImport| {
                 if sess.target.arch == "x86" {
                     (
-                        LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain),
-                        import.ordinal,
+                        common::i686_decorated_name(import, mingw_gnu_toolchain, false),
+                        import.ordinal(),
                     )
                 } else {
-                    (import.name.to_string(), import.ordinal)
+                    (import.name.to_string(), import.ordinal())
                 }
             })
             .collect();
@@ -136,7 +136,8 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
             // that loaded but crashed with an AV upon calling one of the imported
             // functions.  Therefore, use binutils to create the import library instead,
             // by writing a .DEF file to the temp dir and calling binutils's dlltool.
-            let def_file_path = tmpdir.join(format!("{}_imports", lib_name)).with_extension("def");
+            let def_file_path =
+                tmpdir.join(format!("{}{}", lib_name, name_suffix)).with_extension("def");
 
             let def_file_content = format!(
                 "EXPORTS\n{}",
@@ -159,6 +160,9 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
                 }
             };
 
+            // --no-leading-underscore: For the `import_name_type` feature to work, we need to be
+            // able to control the *exact* spelling of each of the symbols that are being imported:
+            // hence we don't want `dlltool` adding leading underscores automatically.
             let dlltool = find_binutils_dlltool(sess);
             let result = std::process::Command::new(dlltool)
                 .args([
@@ -168,6 +172,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
                     lib_name,
                     "-l",
                     output_path.to_str().unwrap(),
+                    "--no-leading-underscore",
                 ])
                 .output();
 
@@ -188,10 +193,10 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
 
             let output_path_z = rustc_fs_util::path_to_c_string(&output_path);
 
-            tracing::trace!("invoking LLVMRustWriteImportLibrary");
-            tracing::trace!("  dll_name {:#?}", dll_name_z);
-            tracing::trace!("  output_path {}", output_path.display());
-            tracing::trace!(
+            trace!("invoking LLVMRustWriteImportLibrary");
+            trace!("  dll_name {:#?}", dll_name_z);
+            trace!("  output_path {}", output_path.display());
+            trace!(
                 "  import names: {}",
                 dll_imports
                     .iter()
@@ -322,22 +327,6 @@ impl<'a> LlvmArchiveBuilder<'a> {
             ret
         }
     }
-
-    fn i686_decorated_name(import: &DllImport, mingw: bool) -> String {
-        let name = import.name;
-        let prefix = if mingw { "" } else { "_" };
-
-        match import.calling_convention {
-            DllCallingConvention::C => format!("{}{}", prefix, name),
-            DllCallingConvention::Stdcall(arg_list_size) => {
-                format!("{}{}@{}", prefix, name, arg_list_size)
-            }
-            DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size),
-            DllCallingConvention::Vectorcall(arg_list_size) => {
-                format!("{}@@{}", name, arg_list_size)
-            }
-        }
-    }
 }
 
 fn string_to_io_error(s: String) -> io::Error {
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index 3731c6bcf..a89df00e2 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -1,15 +1,16 @@
 use crate::back::write::{
     self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers,
 };
-use crate::llvm::archive_ro::ArchiveRO;
 use crate::llvm::{self, build_string, False, True};
 use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm};
+use object::read::archive::ArchiveFile;
 use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared};
 use rustc_codegen_ssa::back::symbol_export;
 use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig};
 use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind};
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::memmap::Mmap;
 use rustc_errors::{FatalError, Handler};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
@@ -17,7 +18,6 @@ use rustc_middle::dep_graph::WorkProduct;
 use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
 use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{self, CrateType, Lto};
-use tracing::{debug, info};
 
 use std::ffi::{CStr, CString};
 use std::fs::File;
@@ -107,14 +107,24 @@ fn prepare_lto(
                     .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
             }
 
-            let archive = ArchiveRO::open(path).expect("wanted an rlib");
+            let archive_data = unsafe {
+                Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib"))
+                    .expect("couldn't map rlib")
+            };
+            let archive = ArchiveFile::parse(&*archive_data).expect("wanted an rlib");
             let obj_files = archive
-                .iter()
-                .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c))))
+                .members()
+                .filter_map(|child| {
+                    child.ok().and_then(|c| {
+                        std::str::from_utf8(c.name()).ok().map(|name| (name.trim(), c))
+                    })
+                })
                 .filter(|&(name, _)| looks_like_rust_object_file(name));
             for (name, child) in obj_files {
                 info!("adding bitcode from {}", name);
-                match get_bitcode_slice_from_object_data(child.data()) {
+                match get_bitcode_slice_from_object_data(
+                    child.data(&*archive_data).expect("corrupt rlib"),
+                ) {
                     Ok(data) => {
                         let module = SerializedModule::FromRlib(data.to_vec());
                         upstream_modules.push((module, CString::new(name).unwrap()));
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index 534d32e8a..a695df840 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -5,7 +5,7 @@ use crate::back::profiling::{
 use crate::base;
 use crate::common;
 use crate::consts;
-use crate::llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic};
+use crate::llvm::{self, DiagnosticInfo, PassManager};
 use crate::llvm_util;
 use crate::type_::Type;
 use crate::LlvmCodegenBackend;
@@ -28,7 +28,6 @@ use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::InnerSpan;
 use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo};
-use tracing::debug;
 
 use libc::{c_char, c_int, c_uint, c_void, size_t};
 use std::ffi::CString;
@@ -304,7 +303,6 @@ impl<'a> DiagnosticHandlers<'a> {
                 remark_passes.as_ptr(),
                 remark_passes.len(),
             );
-            llvm::LLVMRustSetInlineAsmDiagnosticHandler(llcx, inline_asm_handler, data.cast());
             DiagnosticHandlers { data, llcx, old_handler }
         }
     }
@@ -312,9 +310,7 @@ impl<'a> DiagnosticHandlers<'a> {
 
 impl<'a> Drop for DiagnosticHandlers<'a> {
     fn drop(&mut self) {
-        use std::ptr::null_mut;
         unsafe {
-            llvm::LLVMRustSetInlineAsmDiagnosticHandler(self.llcx, inline_asm_handler, null_mut());
             llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler);
             drop(Box::from_raw(self.data));
         }
@@ -342,16 +338,6 @@ fn report_inline_asm(
     cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source);
 }
 
-unsafe extern "C" fn inline_asm_handler(diag: &SMDiagnostic, user: *const c_void, cookie: c_uint) {
-    if user.is_null() {
-        return;
-    }
-    let (cgcx, _) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &Handler));
-
-    let smdiag = llvm::diagnostic::SrcMgrDiagnostic::unpack(diag);
-    report_inline_asm(cgcx, smdiag.message, smdiag.level, cookie, smdiag.source);
-}
-
 unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) {
     if user.is_null() {
         return;
@@ -423,6 +409,14 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
         .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
 }
 
+fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
+    if config.instrument_coverage {
+        Some(CString::new("default_%m_%p.profraw").unwrap())
+    } else {
+        None
+    }
+}
+
 pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     diag_handler: &Handler,
@@ -438,6 +432,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
     let pgo_use_path = get_pgo_use_path(config);
     let pgo_sample_use_path = get_pgo_sample_use_path(config);
     let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
+    let instr_profile_output_path = get_instr_profile_output_path(config);
     // Sanitizer instrumentation is only inserted during the pre-link optimization stage.
     let sanitizer_options = if !is_lto {
         Some(llvm::SanitizerOptions {
@@ -488,6 +483,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
         pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
         pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
         config.instrument_coverage,
+        instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
         config.instrument_gcov,
         pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
         config.debug_info_for_profiling,
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index d3096c73a..63b63c6a1 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -3,7 +3,6 @@ use crate::common::Funclet;
 use crate::context::CodegenCx;
 use crate::llvm::{self, BasicBlock, False};
 use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope};
-use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -28,7 +27,6 @@ use std::ffi::CStr;
 use std::iter;
 use std::ops::Deref;
 use std::ptr;
-use tracing::{debug, instrument};
 
 // All Builders must have an llfn associated with them
 #[must_use]
@@ -726,11 +724,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
-    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
+    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         self.fptoint_sat(false, val, dest_ty)
     }
 
-    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
+    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         self.fptoint_sat(true, val, dest_ty)
     }
 
@@ -1038,25 +1036,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         dst: &'ll Value,
         cmp: &'ll Value,
         src: &'ll Value,
-        mut order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_codegen_ssa::common::AtomicOrdering,
         failure_order: rustc_codegen_ssa::common::AtomicOrdering,
         weak: bool,
     ) -> &'ll Value {
         let weak = if weak { llvm::True } else { llvm::False };
-        if llvm_util::get_version() < (13, 0, 0) {
-            use rustc_codegen_ssa::common::AtomicOrdering::*;
-            // Older llvm has the pre-C++17 restriction on
-            // success and failure memory ordering,
-            // requiring the former to be at least as strong as the latter.
-            // So, for llvm 12, we upgrade the success ordering to a stronger
-            // one if necessary.
-            match (order, failure_order) {
-                (Relaxed, Acquire) => order = Acquire,
-                (Release, Acquire) => order = AcquireRelease,
-                (_, SequentiallyConsistent) => order = SequentiallyConsistent,
-                _ => {}
-            }
-        }
         unsafe {
             llvm::LLVMRustBuildAtomicCmpXchg(
                 self.llbuilder,
@@ -1444,51 +1428,32 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         }
     }
 
-    fn fptoint_sat_broken_in_llvm(&self) -> bool {
-        match self.tcx.sess.target.arch.as_ref() {
-            // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083
-            "riscv64" => llvm_util::get_version() < (13, 0, 0),
-            _ => false,
-        }
-    }
-
-    fn fptoint_sat(
-        &mut self,
-        signed: bool,
-        val: &'ll Value,
-        dest_ty: &'ll Type,
-    ) -> Option<&'ll Value> {
-        if !self.fptoint_sat_broken_in_llvm() {
-            let src_ty = self.cx.val_ty(val);
-            let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector
-            {
-                assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty));
-                (
-                    self.cx.element_type(src_ty),
-                    self.cx.element_type(dest_ty),
-                    Some(self.cx.vector_length(src_ty)),
-                )
-            } else {
-                (src_ty, dest_ty, None)
-            };
-            let float_width = self.cx.float_width(float_ty);
-            let int_width = self.cx.int_width(int_ty);
-
-            let instr = if signed { "fptosi" } else { "fptoui" };
-            let name = if let Some(vector_length) = vector_length {
-                format!(
-                    "llvm.{}.sat.v{}i{}.v{}f{}",
-                    instr, vector_length, int_width, vector_length, float_width
-                )
-            } else {
-                format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width)
-            };
-            let f =
-                self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty));
-            Some(self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None))
+    fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+        let src_ty = self.cx.val_ty(val);
+        let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector {
+            assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty));
+            (
+                self.cx.element_type(src_ty),
+                self.cx.element_type(dest_ty),
+                Some(self.cx.vector_length(src_ty)),
+            )
         } else {
-            None
-        }
+            (src_ty, dest_ty, None)
+        };
+        let float_width = self.cx.float_width(float_ty);
+        let int_width = self.cx.int_width(int_ty);
+
+        let instr = if signed { "fptosi" } else { "fptoui" };
+        let name = if let Some(vector_length) = vector_length {
+            format!(
+                "llvm.{}.sat.v{}i{}.v{}f{}",
+                instr, vector_length, int_width, vector_length, float_width
+            )
+        } else {
+            format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width)
+        };
+        let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty));
+        self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None)
     }
 
     pub(crate) fn landing_pad(
diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs
index 72155d874..b83c1e8f0 100644
--- a/compiler/rustc_codegen_llvm/src/callee.rs
+++ b/compiler/rustc_codegen_llvm/src/callee.rs
@@ -6,11 +6,11 @@
 
 use crate::abi::FnAbiLlvmExt;
 use crate::attributes;
+use crate::common;
 use crate::context::CodegenCx;
 use crate::llvm;
 use crate::value::Value;
 use rustc_codegen_ssa::traits::*;
-use tracing::debug;
 
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
 use rustc_middle::ty::{self, Instance, TypeVisitable};
@@ -79,13 +79,18 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) ->
             llfn
         }
     } else {
-        let llfn = cx.declare_fn(sym, fn_abi);
+        let instance_def_id = instance.def_id();
+        let llfn = if tcx.sess.target.arch == "x86" &&
+            let Some(dllimport) = common::get_dllimport(tcx, instance_def_id, sym)
+        {
+            cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi)
+        } else {
+            cx.declare_fn(sym, fn_abi)
+        };
         debug!("get_fn: not casting pointer!");
 
         attributes::from_fn_attrs(cx, llfn, instance);
 
-        let instance_def_id = instance.def_id();
-
         // Apply an appropriate linkage/visibility value to our item that we
         // just declared.
         //
diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs
index fb4da9a5f..acee9134f 100644
--- a/compiler/rustc_codegen_llvm/src/common.rs
+++ b/compiler/rustc_codegen_llvm/src/common.rs
@@ -10,13 +10,17 @@ use crate::value::Value;
 use rustc_ast::Mutability;
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
+use rustc_hir::def_id::DefId;
 use rustc_middle::bug;
 use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
+use rustc_middle::ty::TyCtxt;
+use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType};
 use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size};
+use rustc_target::spec::Target;
 
 use libc::{c_char, c_uint};
-use tracing::debug;
+use std::fmt::Write;
 
 /*
 * A note on nomenclature of linking: "extern", "foreign", and "upcall".
@@ -211,7 +215,11 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     }
 
     fn const_to_opt_uint(&self, v: &'ll Value) -> Option<u64> {
-        try_as_const_integral(v).map(|v| unsafe { llvm::LLVMConstIntGetZExtValue(v) })
+        try_as_const_integral(v).and_then(|v| unsafe {
+            let mut i = 0u64;
+            let success = llvm::LLVMRustConstIntGetZExtValue(v, &mut i);
+            success.then_some(i)
+        })
     }
 
     fn const_to_opt_u128(&self, v: &'ll Value, sign_ext: bool) -> Option<u128> {
@@ -222,10 +230,6 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         })
     }
 
-    fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value {
-        self.const_undef(self.type_ix(0))
-    }
-
     fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value {
         let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() };
         match cv {
@@ -357,3 +361,74 @@ fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 {
 fn try_as_const_integral(v: &Value) -> Option<&ConstantInt> {
     unsafe { llvm::LLVMIsAConstantInt(v) }
 }
+
+pub(crate) fn get_dllimport<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    id: DefId,
+    name: &str,
+) -> Option<&'tcx DllImport> {
+    tcx.native_library(id)
+        .map(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name))
+        .flatten()
+}
+
+pub(crate) fn is_mingw_gnu_toolchain(target: &Target) -> bool {
+    target.vendor == "pc" && target.os == "windows" && target.env == "gnu" && target.abi.is_empty()
+}
+
+pub(crate) fn i686_decorated_name(
+    dll_import: &DllImport,
+    mingw: bool,
+    disable_name_mangling: bool,
+) -> String {
+    let name = dll_import.name.as_str();
+
+    let (add_prefix, add_suffix) = match dll_import.import_name_type {
+        Some(PeImportNameType::NoPrefix) => (false, true),
+        Some(PeImportNameType::Undecorated) => (false, false),
+        _ => (true, true),
+    };
+
+    // Worst case: +1 for disable name mangling, +1 for prefix, +4 for suffix (@@__).
+    let mut decorated_name = String::with_capacity(name.len() + 6);
+
+    if disable_name_mangling {
+        // LLVM uses a binary 1 ('\x01') prefix to a name to indicate that mangling needs to be disabled.
+        decorated_name.push('\x01');
+    }
+
+    let prefix = if add_prefix && dll_import.is_fn {
+        match dll_import.calling_convention {
+            DllCallingConvention::C | DllCallingConvention::Vectorcall(_) => None,
+            DllCallingConvention::Stdcall(_) => (!mingw
+                || dll_import.import_name_type == Some(PeImportNameType::Decorated))
+            .then_some('_'),
+            DllCallingConvention::Fastcall(_) => Some('@'),
+        }
+    } else if !dll_import.is_fn && !mingw {
+        // For static variables, prefix with '_' on MSVC.
+        Some('_')
+    } else {
+        None
+    };
+    if let Some(prefix) = prefix {
+        decorated_name.push(prefix);
+    }
+
+    decorated_name.push_str(name);
+
+    if add_suffix && dll_import.is_fn {
+        match dll_import.calling_convention {
+            DllCallingConvention::C => {}
+            DllCallingConvention::Stdcall(arg_list_size)
+            | DllCallingConvention::Fastcall(arg_list_size) => {
+                write!(&mut decorated_name, "@{}", arg_list_size).unwrap();
+            }
+            DllCallingConvention::Vectorcall(arg_list_size) => {
+                write!(&mut decorated_name, "@@{}", arg_list_size).unwrap();
+            }
+        }
+    }
+
+    decorated_name
+}
diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index 18467e370..a559f7f3d 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -1,5 +1,5 @@
 use crate::base;
-use crate::common::CodegenCx;
+use crate::common::{self, CodegenCx};
 use crate::debuginfo;
 use crate::llvm::{self, True};
 use crate::llvm_util;
@@ -23,16 +23,15 @@ use rustc_target::abi::{
     AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange,
 };
 use std::ops::Range;
-use tracing::debug;
 
 pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value {
     let alloc = alloc.inner();
-    let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
+    let mut llvals = Vec::with_capacity(alloc.provenance().len() + 1);
     let dl = cx.data_layout();
     let pointer_size = dl.pointer_size.bytes() as usize;
 
-    // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`,
-    // so `range` must be within the bounds of `alloc` and not contain or overlap a relocation.
+    // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range`
+    // must be within the bounds of `alloc` and not contain or overlap a pointer provenance.
     fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>(
         llvals: &mut Vec<&'ll Value>,
         cx: &'a CodegenCx<'ll, 'b>,
@@ -79,12 +78,12 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
     }
 
     let mut next_offset = 0;
-    for &(offset, alloc_id) in alloc.relocations().iter() {
+    for &(offset, alloc_id) in alloc.provenance().iter() {
         let offset = offset.bytes();
         assert_eq!(offset as usize as u64, offset);
         let offset = offset as usize;
         if offset > next_offset {
-            // This `inspect` is okay since we have checked that it is not within a relocation, it
+            // This `inspect` is okay since we have checked that there is no provenance, it
             // is within the bounds of the allocation, and it doesn't affect interpreter execution
             // (we inspect the result after interpreter execution).
             append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset);
@@ -93,7 +92,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
             dl.endian,
             // This `inspect` is okay since it is within the bounds of the allocation, it doesn't
             // affect interpreter execution (we inspect the result after interpreter execution),
-            // and we properly interpret the relocation as a relocation pointer offset.
+            // and we properly interpret the provenance as a relocation pointer offset.
             alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)),
         )
         .expect("const_alloc_to_llvm: could not read relocation pointer")
@@ -121,7 +120,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
     }
     if alloc.len() >= next_offset {
         let range = next_offset..alloc.len();
-        // This `inspect` is okay since we have check that it is after all relocations, it is
+        // This `inspect` is okay since we have check that it is after all provenance, it is
         // within the bounds of the allocation, and it doesn't affect interpreter execution (we
         // inspect the result after interpreter execution).
         append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range);
@@ -160,7 +159,7 @@ fn check_and_apply_linkage<'ll, 'tcx>(
     attrs: &CodegenFnAttrs,
     ty: Ty<'tcx>,
     sym: &str,
-    span_def_id: DefId,
+    def_id: DefId,
 ) -> &'ll Value {
     let llty = cx.layout_of(ty).llvm_type(cx);
     if let Some(linkage) = attrs.linkage {
@@ -175,7 +174,7 @@ fn check_and_apply_linkage<'ll, 'tcx>(
             cx.layout_of(mt.ty).llvm_type(cx)
         } else {
             cx.sess().span_fatal(
-                cx.tcx.def_span(span_def_id),
+                cx.tcx.def_span(def_id),
                 "must have type `*const T` or `*mut T` due to `#[linkage]` attribute",
             )
         };
@@ -194,7 +193,7 @@ fn check_and_apply_linkage<'ll, 'tcx>(
             real_name.push_str(sym);
             let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| {
                 cx.sess().span_fatal(
-                    cx.tcx.def_span(span_def_id),
+                    cx.tcx.def_span(def_id),
                     &format!("symbol `{}` is already defined", &sym),
                 )
             });
@@ -202,6 +201,10 @@ fn check_and_apply_linkage<'ll, 'tcx>(
             llvm::LLVMSetInitializer(g2, g1);
             g2
         }
+    } else if cx.tcx.sess.target.arch == "x86" &&
+        let Some(dllimport) = common::get_dllimport(cx.tcx, def_id, sym)
+    {
+        cx.declare_global(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&cx.tcx.sess.target), true), llty)
     } else {
         // Generate an external declaration.
         // FIXME(nagisa): investigate whether it can be changed into define_global
@@ -475,7 +478,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                 //
                 // We could remove this hack whenever we decide to drop macOS 10.10 support.
                 if self.tcx.sess.target.is_like_osx {
-                    // The `inspect` method is okay here because we checked relocations, and
+                    // The `inspect` method is okay here because we checked for provenance, and
                     // because we are doing this access to inspect the final interpreter state
                     // (not as part of the interpreter execution).
                     //
@@ -483,7 +486,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                     // happens to be zero. Instead, we should only check the value of defined bytes
                     // and set all undefined bytes to zero if this allocation is headed for the
                     // BSS.
-                    let all_bytes_are_zero = alloc.relocations().is_empty()
+                    let all_bytes_are_zero = alloc.provenance().is_empty()
                         && alloc
                             .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len())
                             .iter()
@@ -507,9 +510,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                         section.as_str().as_ptr().cast(),
                         section.as_str().len() as c_uint,
                     );
-                    assert!(alloc.relocations().is_empty());
+                    assert!(alloc.provenance().is_empty());
 
-                    // The `inspect` method is okay here because we checked relocations, and
+                    // The `inspect` method is okay here because we checked for provenance, and
                     // because we are doing this access to inspect the final interpreter state (not
                     // as part of the interpreter execution).
                     let bytes =
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index 5857b83f6..67ffc7cb9 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -142,17 +142,6 @@ pub unsafe fn create_module<'ll>(
 
     let mut target_data_layout = sess.target.data_layout.to_string();
     let llvm_version = llvm_util::get_version();
-    if llvm_version < (13, 0, 0) {
-        if sess.target.arch == "powerpc64" {
-            target_data_layout = target_data_layout.replace("-S128", "");
-        }
-        if sess.target.arch == "wasm32" {
-            target_data_layout = "e-m:e-p:32:32-i64:64-n32:64-S128".to_string();
-        }
-        if sess.target.arch == "wasm64" {
-            target_data_layout = "e-m:e-p:64:64-i64:64-n32:64-S128".to_string();
-        }
-    }
     if llvm_version < (14, 0, 0) {
         if sess.target.llvm_target == "i686-pc-windows-msvc"
             || sess.target.llvm_target == "i586-pc-windows-msvc"
@@ -897,6 +886,9 @@ impl<'ll> CodegenCx<'ll, '_> {
             ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void);
             ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void);
         }
+
+        ifn!("llvm.ptrmask", fn(i8p, t_isize) -> i8p);
+
         None
     }
 
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
index 58f391692..0d1df6fb1 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
@@ -16,8 +16,6 @@ use rustc_middle::ty::TyCtxt;
 
 use std::ffi::CString;
 
-use tracing::debug;
-
 /// Generates and exports the Coverage Map.
 ///
 /// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
index 98ba38356..964a632b6 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
@@ -28,7 +28,6 @@ use std::cell::RefCell;
 use std::ffi::CString;
 
 use std::iter;
-use tracing::debug;
 
 pub mod mapgen;
 
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
index bd84100e0..163ccd946 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@@ -42,7 +42,6 @@ use rustc_span::{self, FileNameDisplayPreference, SourceFile};
 use rustc_symbol_mangling::typeid_for_trait_ref;
 use rustc_target::abi::{Align, Size};
 use smallvec::smallvec;
-use tracing::debug;
 
 use libc::{c_char, c_longlong, c_uint};
 use std::borrow::Cow;
@@ -51,7 +50,6 @@ use std::hash::{Hash, Hasher};
 use std::iter;
 use std::path::{Path, PathBuf};
 use std::ptr;
-use tracing::instrument;
 
 impl PartialEq for llvm::Metadata {
     fn eq(&self, other: &Self) -> bool {
@@ -114,6 +112,7 @@ macro_rules! return_if_di_node_created_in_meantime {
 }
 
 /// Extract size and alignment from a TyAndLayout.
+#[inline]
 fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) {
     (ty_and_layout.size, ty_and_layout.align.abi)
 }
@@ -1499,24 +1498,18 @@ fn vcall_visibility_metadata<'ll, 'tcx>(
         // If there is not LTO and the visibility in public, we have to assume that the vtable can
         // be seen from anywhere. With multiple CGUs, the vtable is quasi-public.
         (Lto::No | Lto::ThinLocal, Visibility::Public, _)
-        | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => {
-            VCallVisibility::Public
-        }
+        | (Lto::No, Visibility::Restricted(_), false) => VCallVisibility::Public,
         // With LTO and a quasi-public visibility, the usages of the functions of the vtable are
         // all known by the `LinkageUnit`.
         // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also
         // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those.
         (Lto::Fat | Lto::Thin, Visibility::Public, _)
-        | (
-            Lto::ThinLocal | Lto::Thin | Lto::Fat,
-            Visibility::Restricted(_) | Visibility::Invisible,
-            false,
-        ) => VCallVisibility::LinkageUnit,
+        | (Lto::ThinLocal | Lto::Thin | Lto::Fat, Visibility::Restricted(_), false) => {
+            VCallVisibility::LinkageUnit
+        }
         // If there is only one CGU, private vtables can only be seen by that CGU/translation unit
         // and therefore we know of all usages of functions in the vtable.
-        (_, Visibility::Restricted(_) | Visibility::Invisible, true) => {
-            VCallVisibility::TranslationUnit
-        }
+        (_, Visibility::Restricted(_), true) => VCallVisibility::TranslationUnit,
     };
 
     let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref);
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
index d6e2c8ccd..129e336c7 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
@@ -1,19 +1,21 @@
 use std::borrow::Cow;
 
 use libc::c_uint;
-use rustc_codegen_ssa::debuginfo::{
-    type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo,
+use rustc_codegen_ssa::{
+    debuginfo::{type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo},
+    traits::ConstMethods,
 };
+
+use rustc_index::vec::IndexVec;
 use rustc_middle::{
     bug,
     ty::{
         self,
         layout::{LayoutOf, TyAndLayout},
-        util::Discr,
-        AdtDef, GeneratorSubsts,
+        AdtDef, GeneratorSubsts, Ty,
     },
 };
-use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants};
+use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants};
 use smallvec::smallvec;
 
 use crate::{
@@ -21,9 +23,9 @@ use crate::{
     debuginfo::{
         metadata::{
             build_field_di_node, closure_saved_names_of_captured_variables,
-            enums::tag_base_type,
-            file_metadata, generator_layout_and_saved_local_names, size_and_align_of,
-            type_map::{self, UniqueTypeId},
+            enums::{tag_base_type, DiscrResult},
+            file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node,
+            type_map::{self, Stub, UniqueTypeId},
             unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA,
             UNKNOWN_LINE_NUMBER,
         },
@@ -35,59 +37,161 @@ use crate::{
     },
 };
 
-/// In CPP-like mode, we generate a union of structs for each variant and an
-/// explicit discriminant field roughly equivalent to the following C/C++ code:
+// The names of the associated constants in each variant wrapper struct.
+// These have to match up with the names being used in `intrinsic.natvis`.
+const ASSOC_CONST_DISCR_NAME: &str = "NAME";
+const ASSOC_CONST_DISCR_EXACT: &str = "DISCR_EXACT";
+const ASSOC_CONST_DISCR_BEGIN: &str = "DISCR_BEGIN";
+const ASSOC_CONST_DISCR_END: &str = "DISCR_END";
+
+const ASSOC_CONST_DISCR128_EXACT_LO: &str = "DISCR128_EXACT_LO";
+const ASSOC_CONST_DISCR128_EXACT_HI: &str = "DISCR128_EXACT_HI";
+const ASSOC_CONST_DISCR128_BEGIN_LO: &str = "DISCR128_BEGIN_LO";
+const ASSOC_CONST_DISCR128_BEGIN_HI: &str = "DISCR128_BEGIN_HI";
+const ASSOC_CONST_DISCR128_END_LO: &str = "DISCR128_END_LO";
+const ASSOC_CONST_DISCR128_END_HI: &str = "DISCR128_END_HI";
+
+// The name of the tag field in the top-level union
+const TAG_FIELD_NAME: &str = "tag";
+const TAG_FIELD_NAME_128_LO: &str = "tag128_lo";
+const TAG_FIELD_NAME_128_HI: &str = "tag128_hi";
+
+// We assign a "virtual" discriminant value to the sole variant of
+// a single-variant enum.
+const SINGLE_VARIANT_VIRTUAL_DISR: u64 = 0;
+
+/// In CPP-like mode, we generate a union with a field for each variant and an
+/// explicit tag field. The field of each variant has a struct type
+/// that encodes the discrimiant of the variant and it's data layout.
+/// The union also has a nested enumeration type that is only used for encoding
+/// variant names in an efficient way. Its enumerator values do _not_ correspond
+/// to the enum's discriminant values.
+/// It's roughly equivalent to the following C/C++ code:
 ///
 /// ```c
-/// union enum$<{fully-qualified-name}> {
-///   struct {variant 0 name} {
-///     <variant 0 fields>
+/// union enum2$<{fully-qualified-name}> {
+///   struct Variant0 {
+///     struct {name-of-variant-0} {
+///        <variant 0 fields>
+///     } value;
+///
+///     static VariantNames NAME = {name-of-variant-0};
+///     static int_type DISCR_EXACT = {discriminant-of-variant-0};
 ///   } variant0;
+///
 ///   <other variant structs>
-///   {name} discriminant;
+///
+///   int_type tag;
+///
+///   enum VariantNames {
+///      <name-of-variant-0> = 0, // The numeric values are variant index,
+///      <name-of-variant-1> = 1, // not discriminant values.
+///      <name-of-variant-2> = 2,
+///      ...
+///   }
 /// }
 /// ```
 ///
-/// As you can see, the type name is wrapped `enum$`. This way we can have a
-/// single NatVis rule for handling all enums.
+/// As you can see, the type name is wrapped in `enum2$<_>`. This way we can
+/// have a single NatVis rule for handling all enums. The `2` in `enum2$<_>`
+/// is an encoding version tag, so that debuggers can decide to decode this
+/// differently than the previous `enum$<_>` encoding emitted by earlier
+/// compiler versions.
 ///
-/// At the LLVM IR level this looks like
+/// Niche-tag enums have one special variant, usually called the
+/// "untagged variant". This variant has a field that
+/// doubles as the tag of the enum. The variant is active when the value of
+/// that field is within a pre-defined range. Therefore the variant struct
+/// has a `DISCR_BEGIN` and `DISCR_END` field instead of `DISCR_EXACT` in
+/// that case. Both `DISCR_BEGIN` and `DISCR_END` are inclusive bounds.
+/// Note that these ranges can wrap around, so that `DISCR_END < DISCR_BEGIN`.
 ///
-/// ```txt
-///       DW_TAG_union_type              (top-level type for enum)
-///         DW_TAG_member                    (member for variant 1)
-///         DW_TAG_member                    (member for variant 2)
-///         DW_TAG_member                    (member for variant 3)
-///         DW_TAG_structure_type            (type of variant 1)
-///         DW_TAG_structure_type            (type of variant 2)
-///         DW_TAG_structure_type            (type of variant 3)
-///         DW_TAG_enumeration_type          (type of tag)
-/// ```
+/// Single-variant enums don't actually have a tag field. In this case we
+/// emit a static tag field (that always has the value 0) so we can use the
+/// same representation (and NatVis).
 ///
-/// The above encoding applies for enums with a direct tag. For niche-tag we have to do things
-/// differently in order to allow a NatVis visualizer to extract all the information needed:
-/// We generate a union of two fields, one for the dataful variant
-/// and one that just points to the discriminant (which is some field within the dataful variant).
-/// We also create a DW_TAG_enumeration_type DIE that contains tag values for the non-dataful
-/// variants and make the discriminant field that type. We then use NatVis to render the enum type
-/// correctly in Windbg/VS. This will generate debuginfo roughly equivalent to the following C:
+/// For niche-layout enums it's possible to have a 128-bit tag. NatVis, VS, and
+/// WinDbg (the main targets for CPP-like debuginfo at the moment) don't support
+/// 128-bit integers, so all values involved get split into two 64-bit fields.
+/// Instead of the `tag` field, we generate two fields `tag128_lo` and `tag128_hi`,
+/// Instead of `DISCR_EXACT`, we generate `DISCR128_EXACT_LO` and `DISCR128_EXACT_HI`,
+/// and so on.
 ///
-/// ```c
-/// union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> {
-///   struct <dataful variant name> {
-///     <fields in dataful variant>
-///   } dataful_variant;
-///   enum Discriminant$ {
-///     <non-dataful variants>
-///   } discriminant;
+///
+/// The following pseudocode shows how to decode an enum value in a debugger:
+///
+/// ```text
+///
+/// fn find_active_variant(enum_value) -> (VariantName, VariantValue) {
+///     let is_128_bit = enum_value.has_field("tag128_lo");
+///
+///     if !is_128_bit {
+///         // Note: `tag` can be a static field for enums with only one
+///         //       inhabited variant.
+///         let tag = enum_value.field("tag").value;
+///
+///         // For each variant, check if it is a match. Only one of them will match,
+///         // so if we find it we can return it immediately.
+///         for variant_field in enum_value.fields().filter(|f| f.name.starts_with("variant")) {
+///             if variant_field.has_field("DISCR_EXACT") {
+///                 // This variant corresponds to a single tag value
+///                 if variant_field.field("DISCR_EXACT").value == tag {
+///                     return (variant_field.field("NAME"), variant_field.value);
+///                 }
+///             } else {
+///                 // This is a range variant
+///                 let begin = variant_field.field("DISCR_BEGIN");
+///                 let end = variant_field.field("DISCR_END");
+///
+///                 if is_in_range(tag, begin, end) {
+///                     return (variant_field.field("NAME"), variant_field.value);
+///                 }
+///             }
+///         }
+///     } else {
+///         // Basically the same as with smaller tags, we just have to
+///         // stitch the values together.
+///         let tag: u128 = (enum_value.field("tag128_lo").value as u128) |
+///                         (enum_value.field("tag128_hi").value as u128 << 64);
+///
+///         for variant_field in enum_value.fields().filter(|f| f.name.starts_with("variant")) {
+///             if variant_field.has_field("DISCR128_EXACT_LO") {
+///                 let discr_exact = (variant_field.field("DISCR128_EXACT_LO" as u128) |
+///                                   (variant_field.field("DISCR128_EXACT_HI") as u128 << 64);
+///
+///                 // This variant corresponds to a single tag value
+///                 if discr_exact.value == tag {
+///                     return (variant_field.field("NAME"), variant_field.value);
+///                 }
+///             } else {
+///                 // This is a range variant
+///                 let begin = (variant_field.field("DISCR128_BEGIN_LO").value as u128) |
+///                             (variant_field.field("DISCR128_BEGIN_HI").value as u128 << 64);
+///                 let end = (variant_field.field("DISCR128_END_LO").value as u128) |
+///                           (variant_field.field("DISCR128_END_HI").value as u128 << 64);
+///
+///                 if is_in_range(tag, begin, end) {
+///                     return (variant_field.field("NAME"), variant_field.value);
+///                 }
+///             }
+///         }
+///     }
+///
+///     // We should have found an active variant at this point.
+///     unreachable!();
 /// }
-/// ```
 ///
-/// The NatVis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>`
-/// and evaluates `this.discriminant`. If the value is between the min niche and max
-/// niche, then the enum is in the dataful variant and `this.dataful_variant` is
-/// rendered. Otherwise, the enum is in one of the non-dataful variants. In that
-/// case, we just need to render the name of the `this.discriminant` enum.
+/// // Check if a value is within the given range
+/// // (where the range might wrap around the value space)
+/// fn is_in_range(value, start, end) -> bool {
+///     if start < end {
+///         value >= start && value <= end
+///     } else {
+///         value >= start || value <= end
+///     }
+/// }
+///
+/// ```
 pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     unique_type_id: UniqueTypeId<'tcx>,
@@ -135,27 +239,28 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
                     ref variants,
                     tag_field,
                     ..
-                } => build_union_fields_for_direct_tag_enum(
+                } => build_union_fields_for_enum(
                     cx,
                     enum_adt_def,
                     enum_type_and_layout,
                     enum_type_di_node,
-                    &mut variants.indices(),
+                    variants.indices(),
                     tag_field,
+                    None,
                 ),
                 Variants::Multiple {
-                    tag_encoding: TagEncoding::Niche { dataful_variant, .. },
+                    tag_encoding: TagEncoding::Niche { untagged_variant, .. },
                     ref variants,
                     tag_field,
                     ..
-                } => build_union_fields_for_niche_tag_enum(
+                } => build_union_fields_for_enum(
                     cx,
                     enum_adt_def,
                     enum_type_and_layout,
                     enum_type_di_node,
-                    dataful_variant,
-                    &mut variants.indices(),
+                    variants.indices(),
                     tag_field,
+                    Some(untagged_variant),
                 ),
             }
         },
@@ -217,137 +322,344 @@ fn build_single_variant_union_fields<'ll, 'tcx>(
     let variant_layout = enum_type_and_layout.for_variant(cx, variant_index);
     let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node(
         cx,
-        enum_type_and_layout.ty,
+        enum_type_and_layout,
         enum_type_di_node,
         variant_index,
         enum_adt_def.variant(variant_index),
         variant_layout,
     );
 
-    // NOTE: The field name of the union is the same as the variant name, not "variant0".
-    let variant_name = enum_adt_def.variant(variant_index).name.as_str();
+    let tag_base_type = cx.tcx.types.u32;
+    let tag_base_type_di_node = type_di_node(cx, tag_base_type);
+    let tag_base_type_align = cx.align_of(tag_base_type);
+
+    let variant_names_type_di_node = build_variant_names_type_di_node(
+        cx,
+        enum_type_di_node,
+        std::iter::once((
+            variant_index,
+            Cow::from(enum_adt_def.variant(variant_index).name.as_str()),
+        )),
+    );
 
-    smallvec![build_field_di_node(
+    let variant_struct_type_wrapper_di_node = build_variant_struct_wrapper_type_di_node(
         cx,
+        enum_type_and_layout,
         enum_type_di_node,
-        variant_name,
-        // NOTE: We use the size and align of the entire type, not from variant_layout
-        //       since the later is sometimes smaller (if it has fewer fields).
-        size_and_align_of(enum_type_and_layout),
-        Size::ZERO,
-        DIFlags::FlagZero,
+        variant_index,
+        None,
         variant_struct_type_di_node,
-    )]
+        variant_names_type_di_node,
+        tag_base_type_di_node,
+        tag_base_type,
+        DiscrResult::NoDiscriminant,
+    );
+
+    smallvec![
+        build_field_di_node(
+            cx,
+            enum_type_di_node,
+            &variant_union_field_name(variant_index),
+            // NOTE: We use the size and align of the entire type, not from variant_layout
+            //       since the later is sometimes smaller (if it has fewer fields).
+            size_and_align_of(enum_type_and_layout),
+            Size::ZERO,
+            DIFlags::FlagZero,
+            variant_struct_type_wrapper_di_node,
+        ),
+        unsafe {
+            llvm::LLVMRustDIBuilderCreateStaticMemberType(
+                DIB(cx),
+                enum_type_di_node,
+                TAG_FIELD_NAME.as_ptr().cast(),
+                TAG_FIELD_NAME.len(),
+                unknown_file_metadata(cx),
+                UNKNOWN_LINE_NUMBER,
+                variant_names_type_di_node,
+                DIFlags::FlagZero,
+                Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)),
+                tag_base_type_align.bits() as u32,
+            )
+        }
+    ]
 }
 
-fn build_union_fields_for_direct_tag_enum<'ll, 'tcx>(
+fn build_union_fields_for_enum<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     enum_adt_def: AdtDef<'tcx>,
     enum_type_and_layout: TyAndLayout<'tcx>,
     enum_type_di_node: &'ll DIType,
-    variant_indices: &mut dyn Iterator<Item = VariantIdx>,
+    variant_indices: impl Iterator<Item = VariantIdx> + Clone,
     tag_field: usize,
+    untagged_variant_index: Option<VariantIdx>,
 ) -> SmallVec<&'ll DIType> {
+    let tag_base_type = super::tag_base_type(cx, enum_type_and_layout);
+
+    let variant_names_type_di_node = build_variant_names_type_di_node(
+        cx,
+        enum_type_di_node,
+        variant_indices.clone().map(|variant_index| {
+            let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str());
+            (variant_index, variant_name)
+        }),
+    );
+
     let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_indices
         .map(|variant_index| {
             let variant_layout = enum_type_and_layout.for_variant(cx, variant_index);
 
+            let variant_def = enum_adt_def.variant(variant_index);
+
+            let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node(
+                cx,
+                enum_type_and_layout,
+                enum_type_di_node,
+                variant_index,
+                variant_def,
+                variant_layout,
+            );
+
             VariantFieldInfo {
                 variant_index,
-                variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node(
-                    cx,
-                    enum_type_and_layout.ty,
-                    enum_type_di_node,
-                    variant_index,
-                    enum_adt_def.variant(variant_index),
-                    variant_layout,
-                ),
+                variant_struct_type_di_node,
                 source_info: None,
+                discr: super::compute_discriminant_value(cx, enum_type_and_layout, variant_index),
             }
         })
         .collect();
 
-    let discr_type_name = cx.tcx.item_name(enum_adt_def.did());
-    let tag_base_type = super::tag_base_type(cx, enum_type_and_layout);
-    let discr_type_di_node = super::build_enumeration_type_di_node(
-        cx,
-        discr_type_name.as_str(),
-        tag_base_type,
-        &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| {
-            (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str()))
-        }),
-        enum_type_di_node,
-    );
-
     build_union_fields_for_direct_tag_enum_or_generator(
         cx,
         enum_type_and_layout,
         enum_type_di_node,
         &variant_field_infos,
-        discr_type_di_node,
+        variant_names_type_di_node,
+        tag_base_type,
         tag_field,
+        untagged_variant_index,
     )
 }
 
-fn build_union_fields_for_niche_tag_enum<'ll, 'tcx>(
+// The base type of the VariantNames DW_AT_enumeration_type is always the same.
+// It has nothing to do with the tag of the enum and just has to be big enough
+// to hold all variant names.
+fn variant_names_enum_base_type<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> Ty<'tcx> {
+    cx.tcx.types.u32
+}
+
+/// This function builds a DW_AT_enumeration_type that contains an entry for
+/// each variant. Note that this has nothing to do with the discriminant. The
+/// numeric value of each enumerator corresponds to the variant index. The
+/// type is only used for efficiently encoding the name of each variant in
+/// debuginfo.
+fn build_variant_names_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
-    enum_adt_def: AdtDef<'tcx>,
-    enum_type_and_layout: TyAndLayout<'tcx>,
-    enum_type_di_node: &'ll DIType,
-    dataful_variant_index: VariantIdx,
-    variant_indices: &mut dyn Iterator<Item = VariantIdx>,
-    tag_field: usize,
-) -> SmallVec<&'ll DIType> {
-    let dataful_variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node(
+    containing_scope: &'ll DIType,
+    variants: impl Iterator<Item = (VariantIdx, Cow<'tcx, str>)>,
+) -> &'ll DIType {
+    // Create an enumerator for each variant.
+    super::build_enumeration_type_di_node(
         cx,
-        enum_type_and_layout.ty,
-        enum_type_di_node,
-        dataful_variant_index,
-        &enum_adt_def.variant(dataful_variant_index),
-        enum_type_and_layout.for_variant(cx, dataful_variant_index),
-    );
+        "VariantNames",
+        variant_names_enum_base_type(cx),
+        variants.map(|(variant_index, variant_name)| (variant_name, variant_index.as_u32() as u64)),
+        containing_scope,
+    )
+}
 
-    let tag_base_type = super::tag_base_type(cx, enum_type_and_layout);
-    // Create an DW_TAG_enumerator for each variant except the dataful one.
-    let discr_type_di_node = super::build_enumeration_type_di_node(
+fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    enum_or_generator_type_and_layout: TyAndLayout<'tcx>,
+    enum_or_generator_type_di_node: &'ll DIType,
+    variant_index: VariantIdx,
+    untagged_variant_index: Option<VariantIdx>,
+    variant_struct_type_di_node: &'ll DIType,
+    variant_names_type_di_node: &'ll DIType,
+    tag_base_type_di_node: &'ll DIType,
+    tag_base_type: Ty<'tcx>,
+    discr: DiscrResult,
+) -> &'ll DIType {
+    type_map::build_type_with_children(
         cx,
-        "Discriminant$",
-        tag_base_type,
-        &mut variant_indices.filter_map(|variant_index| {
-            if let Some(discr_val) =
-                super::compute_discriminant_value(cx, enum_type_and_layout, variant_index)
-            {
-                let discr = Discr { val: discr_val as u128, ty: tag_base_type };
-                let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str());
-                Some((discr, variant_name))
-            } else {
-                debug_assert_eq!(variant_index, dataful_variant_index);
-                None
-            }
-        }),
-        enum_type_di_node,
-    );
-
-    smallvec![
-        build_field_di_node(
-            cx,
-            enum_type_di_node,
-            "dataful_variant",
-            size_and_align_of(enum_type_and_layout),
-            Size::ZERO,
-            DIFlags::FlagZero,
-            dataful_variant_struct_type_di_node,
-        ),
-        build_field_di_node(
+        type_map::stub(
             cx,
-            enum_type_di_node,
-            "discriminant",
-            cx.size_and_align_of(tag_base_type),
-            enum_type_and_layout.fields.offset(tag_field),
+            Stub::Struct,
+            UniqueTypeId::for_enum_variant_struct_type_wrapper(
+                cx.tcx,
+                enum_or_generator_type_and_layout.ty,
+                variant_index,
+            ),
+            &variant_struct_wrapper_type_name(variant_index),
+            // NOTE: We use size and align of enum_type, not from variant_layout:
+            size_and_align_of(enum_or_generator_type_and_layout),
+            Some(enum_or_generator_type_di_node),
             DIFlags::FlagZero,
-            discr_type_di_node,
         ),
-    ]
+        |cx, wrapper_struct_type_di_node| {
+            enum DiscrKind {
+                Exact(u64),
+                Exact128(u128),
+                Range(u64, u64),
+                Range128(u128, u128),
+            }
+
+            let (tag_base_type_size, tag_base_type_align) = cx.size_and_align_of(tag_base_type);
+            let is_128_bits = tag_base_type_size.bits() > 64;
+
+            let discr = match discr {
+                DiscrResult::NoDiscriminant => DiscrKind::Exact(SINGLE_VARIANT_VIRTUAL_DISR),
+                DiscrResult::Value(discr_val) => {
+                    if is_128_bits {
+                        DiscrKind::Exact128(discr_val)
+                    } else {
+                        debug_assert_eq!(discr_val, discr_val as u64 as u128);
+                        DiscrKind::Exact(discr_val as u64)
+                    }
+                }
+                DiscrResult::Range(min, max) => {
+                    assert_eq!(Some(variant_index), untagged_variant_index);
+                    if is_128_bits {
+                        DiscrKind::Range128(min, max)
+                    } else {
+                        debug_assert_eq!(min, min as u64 as u128);
+                        debug_assert_eq!(max, max as u64 as u128);
+                        DiscrKind::Range(min as u64, max as u64)
+                    }
+                }
+            };
+
+            let mut fields = SmallVec::new();
+
+            // We always have a field for the value
+            fields.push(build_field_di_node(
+                cx,
+                wrapper_struct_type_di_node,
+                "value",
+                size_and_align_of(enum_or_generator_type_and_layout),
+                Size::ZERO,
+                DIFlags::FlagZero,
+                variant_struct_type_di_node,
+            ));
+
+            let build_assoc_const =
+                |name: &str, type_di_node: &'ll DIType, value: u64, align: Align| unsafe {
+                    llvm::LLVMRustDIBuilderCreateStaticMemberType(
+                        DIB(cx),
+                        wrapper_struct_type_di_node,
+                        name.as_ptr().cast(),
+                        name.len(),
+                        unknown_file_metadata(cx),
+                        UNKNOWN_LINE_NUMBER,
+                        type_di_node,
+                        DIFlags::FlagZero,
+                        Some(cx.const_u64(value)),
+                        align.bits() as u32,
+                    )
+                };
+
+            // We also always have an associated constant for the discriminant value
+            // of the variant.
+            fields.push(build_assoc_const(
+                ASSOC_CONST_DISCR_NAME,
+                variant_names_type_di_node,
+                variant_index.as_u32() as u64,
+                cx.align_of(variant_names_enum_base_type(cx)),
+            ));
+
+            // Emit the discriminant value (or range) corresponding to the variant.
+            match discr {
+                DiscrKind::Exact(discr_val) => {
+                    fields.push(build_assoc_const(
+                        ASSOC_CONST_DISCR_EXACT,
+                        tag_base_type_di_node,
+                        discr_val,
+                        tag_base_type_align,
+                    ));
+                }
+                DiscrKind::Exact128(discr_val) => {
+                    let align = cx.align_of(cx.tcx.types.u64);
+                    let type_di_node = type_di_node(cx, cx.tcx.types.u64);
+                    let Split128 { hi, lo } = split_128(discr_val);
+
+                    fields.push(build_assoc_const(
+                        ASSOC_CONST_DISCR128_EXACT_LO,
+                        type_di_node,
+                        lo,
+                        align,
+                    ));
+
+                    fields.push(build_assoc_const(
+                        ASSOC_CONST_DISCR128_EXACT_HI,
+                        type_di_node,
+                        hi,
+                        align,
+                    ));
+                }
+                DiscrKind::Range(begin, end) => {
+                    fields.push(build_assoc_const(
+                        ASSOC_CONST_DISCR_BEGIN,
+                        tag_base_type_di_node,
+                        begin,
+                        tag_base_type_align,
+                    ));
+
+                    fields.push(build_assoc_const(
+                        ASSOC_CONST_DISCR_END,
+                        tag_base_type_di_node,
+                        end,
+                        tag_base_type_align,
+                    ));
+                }
+                DiscrKind::Range128(begin, end) => {
+                    let align = cx.align_of(cx.tcx.types.u64);
+                    let type_di_node = type_di_node(cx, cx.tcx.types.u64);
+                    let Split128 { hi: begin_hi, lo: begin_lo } = split_128(begin);
+                    let Split128 { hi: end_hi, lo: end_lo } = split_128(end);
+
+                    fields.push(build_assoc_const(
+                        ASSOC_CONST_DISCR128_BEGIN_HI,
+                        type_di_node,
+                        begin_hi,
+                        align,
+                    ));
+
+                    fields.push(build_assoc_const(
+                        ASSOC_CONST_DISCR128_BEGIN_LO,
+                        type_di_node,
+                        begin_lo,
+                        align,
+                    ));
+
+                    fields.push(build_assoc_const(
+                        ASSOC_CONST_DISCR128_END_HI,
+                        type_di_node,
+                        end_hi,
+                        align,
+                    ));
+
+                    fields.push(build_assoc_const(
+                        ASSOC_CONST_DISCR128_END_LO,
+                        type_di_node,
+                        end_lo,
+                        align,
+                    ));
+                }
+            }
+
+            fields
+        },
+        NO_GENERICS,
+    )
+    .di_node
+}
+
+struct Split128 {
+    hi: u64,
+    lo: u64,
+}
+
+fn split_128(value: u128) -> Split128 {
+    Split128 { hi: (value >> 64) as u64, lo: value as u64 }
 }
 
 fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>(
@@ -369,6 +681,29 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>(
 
     let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id);
     let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx);
+    let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len();
+
+    let tag_base_type = tag_base_type(cx, generator_type_and_layout);
+
+    let variant_names_type_di_node = build_variant_names_type_di_node(
+        cx,
+        generator_type_di_node,
+        variant_range
+            .clone()
+            .map(|variant_index| (variant_index, GeneratorSubsts::variant_name(variant_index))),
+    );
+
+    let discriminants: IndexVec<VariantIdx, DiscrResult> = {
+        let discriminants_iter = generator_substs.discriminants(generator_def_id, cx.tcx);
+        let mut discriminants: IndexVec<VariantIdx, DiscrResult> =
+            IndexVec::with_capacity(variant_count);
+        for (variant_index, discr) in discriminants_iter {
+            // Assert that the index in the IndexMap matches up with the given VariantIdx.
+            assert_eq!(variant_index, discriminants.next_index());
+            discriminants.push(DiscrResult::Value(discr.val));
+        }
+        discriminants
+    };
 
     // Build the type node for each field.
     let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_range
@@ -391,29 +726,24 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>(
                 None
             };
 
-            VariantFieldInfo { variant_index, variant_struct_type_di_node, source_info }
+            VariantFieldInfo {
+                variant_index,
+                variant_struct_type_di_node,
+                source_info,
+                discr: discriminants[variant_index],
+            }
         })
         .collect();
 
-    let tag_base_type = tag_base_type(cx, generator_type_and_layout);
-    let discr_type_name = "Discriminant$";
-    let discr_type_di_node = super::build_enumeration_type_di_node(
-        cx,
-        discr_type_name,
-        tag_base_type,
-        &mut generator_substs
-            .discriminants(generator_def_id, cx.tcx)
-            .map(|(variant_index, discr)| (discr, GeneratorSubsts::variant_name(variant_index))),
-        generator_type_di_node,
-    );
-
     build_union_fields_for_direct_tag_enum_or_generator(
         cx,
         generator_type_and_layout,
         generator_type_di_node,
         &variant_field_infos[..],
-        discr_type_di_node,
+        variant_names_type_di_node,
+        tag_base_type,
         tag_field,
+        None,
     )
 }
 
@@ -425,8 +755,11 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>(
     enum_type_di_node: &'ll DIType,
     variant_field_infos: &[VariantFieldInfo<'ll>],
     discr_type_di_node: &'ll DIType,
+    tag_base_type: Ty<'tcx>,
     tag_field: usize,
+    untagged_variant_index: Option<VariantIdx>,
 ) -> SmallVec<&'ll DIType> {
+    let tag_base_type_di_node = type_di_node(cx, tag_base_type);
     let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1);
 
     // We create a field in the union for each variant ...
@@ -438,6 +771,19 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>(
         let field_name = variant_union_field_name(variant_member_info.variant_index);
         let (size, align) = size_and_align_of(enum_type_and_layout);
 
+        let variant_struct_type_wrapper = build_variant_struct_wrapper_type_di_node(
+            cx,
+            enum_type_and_layout,
+            enum_type_di_node,
+            variant_member_info.variant_index,
+            untagged_variant_index,
+            variant_member_info.variant_struct_type_di_node,
+            discr_type_di_node,
+            tag_base_type_di_node,
+            tag_base_type,
+            variant_member_info.discr,
+        );
+
         // We use LLVMRustDIBuilderCreateMemberType() member type directly because
         // the build_field_di_node() function does not support specifying a source location,
         // which is something that we don't do anywhere else.
@@ -456,7 +802,7 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>(
                 // Union fields are always at offset zero
                 Size::ZERO.bits(),
                 DIFlags::FlagZero,
-                variant_member_info.variant_struct_type_di_node,
+                variant_struct_type_wrapper,
             )
         }
     }));
@@ -466,16 +812,53 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>(
         cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout))
     );
 
-    // ... and a field for the discriminant.
-    unions_fields.push(build_field_di_node(
-        cx,
-        enum_type_di_node,
-        "discriminant",
-        cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty),
-        enum_type_and_layout.fields.offset(tag_field),
-        DIFlags::FlagZero,
-        discr_type_di_node,
-    ));
+    // ... and a field for the tag. If the tag is 128 bits wide, this will actually
+    // be two 64-bit fields.
+    let is_128_bits = cx.size_of(tag_base_type).bits() > 64;
+
+    if is_128_bits {
+        let type_di_node = type_di_node(cx, cx.tcx.types.u64);
+        let size_and_align = cx.size_and_align_of(cx.tcx.types.u64);
+
+        let (lo_offset, hi_offset) = match cx.tcx.data_layout.endian {
+            Endian::Little => (0, 8),
+            Endian::Big => (8, 0),
+        };
+
+        let tag_field_offset = enum_type_and_layout.fields.offset(tag_field).bytes();
+        let lo_offset = Size::from_bytes(tag_field_offset + lo_offset);
+        let hi_offset = Size::from_bytes(tag_field_offset + hi_offset);
+
+        unions_fields.push(build_field_di_node(
+            cx,
+            enum_type_di_node,
+            TAG_FIELD_NAME_128_LO,
+            size_and_align,
+            lo_offset,
+            DIFlags::FlagZero,
+            type_di_node,
+        ));
+
+        unions_fields.push(build_field_di_node(
+            cx,
+            enum_type_di_node,
+            TAG_FIELD_NAME_128_HI,
+            size_and_align,
+            hi_offset,
+            DIFlags::FlagZero,
+            type_di_node,
+        ));
+    } else {
+        unions_fields.push(build_field_di_node(
+            cx,
+            enum_type_di_node,
+            TAG_FIELD_NAME,
+            cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty),
+            enum_type_and_layout.fields.offset(tag_field),
+            DIFlags::FlagZero,
+            tag_base_type_di_node,
+        ));
+    }
 
     unions_fields
 }
@@ -485,6 +868,7 @@ struct VariantFieldInfo<'ll> {
     variant_index: VariantIdx,
     variant_struct_type_di_node: &'ll DIType,
     source_info: Option<(&'ll DIFile, c_uint)>,
+    discr: DiscrResult,
 }
 
 fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> {
@@ -512,3 +896,29 @@ fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> {
         .map(|&s| Cow::from(s))
         .unwrap_or_else(|| format!("variant{}", variant_index.as_usize()).into())
 }
+
+fn variant_struct_wrapper_type_name(variant_index: VariantIdx) -> Cow<'static, str> {
+    const PRE_ALLOCATED: [&str; 16] = [
+        "Variant0",
+        "Variant1",
+        "Variant2",
+        "Variant3",
+        "Variant4",
+        "Variant5",
+        "Variant6",
+        "Variant7",
+        "Variant8",
+        "Variant9",
+        "Variant10",
+        "Variant11",
+        "Variant12",
+        "Variant13",
+        "Variant14",
+        "Variant15",
+    ];
+
+    PRE_ALLOCATED
+        .get(variant_index.as_usize())
+        .map(|&s| Cow::from(s))
+        .unwrap_or_else(|| format!("Variant{}", variant_index.as_usize()).into())
+}
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
index 73e01d045..14044d0f9 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
@@ -10,7 +10,6 @@ use rustc_middle::{
     ty::{
         self,
         layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout},
-        util::Discr,
         AdtDef, GeneratorSubsts, Ty, VariantDef,
     },
 };
@@ -90,8 +89,11 @@ fn build_c_style_enum_di_node<'ll, 'tcx>(
             cx,
             &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false),
             tag_base_type(cx, enum_type_and_layout),
-            &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| {
-                (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str()))
+            enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| {
+                let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str());
+                // Is there anything we can do to support 128-bit C-Style enums?
+                let value = discr.val as u64;
+                (name, value)
             }),
             containing_scope,
         ),
@@ -152,7 +154,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     type_name: &str,
     base_type: Ty<'tcx>,
-    variants: &mut dyn Iterator<Item = (Discr<'tcx>, Cow<'tcx, str>)>,
+    enumerators: impl Iterator<Item = (Cow<'tcx, str>, u64)>,
     containing_scope: &'ll DIType,
 ) -> &'ll DIType {
     let is_unsigned = match base_type.kind() {
@@ -161,18 +163,15 @@ fn build_enumeration_type_di_node<'ll, 'tcx>(
         _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."),
     };
 
-    let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = variants
-        .map(|(discr, variant_name)| {
-            unsafe {
-                Some(llvm::LLVMRustDIBuilderCreateEnumerator(
-                    DIB(cx),
-                    variant_name.as_ptr().cast(),
-                    variant_name.len(),
-                    // FIXME: what if enumeration has i128 discriminant?
-                    discr.val as i64,
-                    is_unsigned,
-                ))
-            }
+    let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = enumerators
+        .map(|(name, value)| unsafe {
+            Some(llvm::LLVMRustDIBuilderCreateEnumerator(
+                DIB(cx),
+                name.as_ptr().cast(),
+                name.len(),
+                value as i64,
+                is_unsigned,
+            ))
         })
         .collect();
 
@@ -247,23 +246,27 @@ fn build_enumeration_type_di_node<'ll, 'tcx>(
 /// and a DW_TAG_member for each field (but not the discriminant).
 fn build_enum_variant_struct_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
-    enum_type: Ty<'tcx>,
+    enum_type_and_layout: TyAndLayout<'tcx>,
     enum_type_di_node: &'ll DIType,
     variant_index: VariantIdx,
     variant_def: &VariantDef,
     variant_layout: TyAndLayout<'tcx>,
 ) -> &'ll DIType {
-    debug_assert_eq!(variant_layout.ty, enum_type);
+    debug_assert_eq!(variant_layout.ty, enum_type_and_layout.ty);
 
     type_map::build_type_with_children(
         cx,
         type_map::stub(
             cx,
             Stub::Struct,
-            UniqueTypeId::for_enum_variant_struct_type(cx.tcx, enum_type, variant_index),
+            UniqueTypeId::for_enum_variant_struct_type(
+                cx.tcx,
+                enum_type_and_layout.ty,
+                variant_index,
+            ),
             variant_def.name.as_str(),
             // NOTE: We use size and align of enum_type, not from variant_layout:
-            cx.size_and_align_of(enum_type),
+            size_and_align_of(enum_type_and_layout),
             Some(enum_type_di_node),
             DIFlags::FlagZero,
         ),
@@ -290,9 +293,9 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>(
                         type_di_node(cx, field_layout.ty),
                     )
                 })
-                .collect()
+                .collect::<SmallVec<_>>()
         },
-        |cx| build_generic_type_param_di_nodes(cx, enum_type),
+        |cx| build_generic_type_param_di_nodes(cx, enum_type_and_layout.ty),
     )
     .di_node
 }
@@ -398,39 +401,60 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>(
     .di_node
 }
 
+#[derive(Copy, Clone)]
+enum DiscrResult {
+    NoDiscriminant,
+    Value(u128),
+    Range(u128, u128),
+}
+
+impl DiscrResult {
+    fn opt_single_val(&self) -> Option<u128> {
+        if let Self::Value(d) = *self { Some(d) } else { None }
+    }
+}
+
 /// Returns the discriminant value corresponding to the variant index.
 ///
 /// Will return `None` if there is less than two variants (because then the enum won't have)
-/// a tag, and if this is the dataful variant of a niche-layout enum (because then there is no
+/// a tag, and if this is the untagged variant of a niche-layout enum (because then there is no
 /// single discriminant value).
 fn compute_discriminant_value<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     enum_type_and_layout: TyAndLayout<'tcx>,
     variant_index: VariantIdx,
-) -> Option<u64> {
+) -> DiscrResult {
     match enum_type_and_layout.layout.variants() {
-        &Variants::Single { .. } => None,
-        &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => Some(
-            enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val
-                as u64,
+        &Variants::Single { .. } => DiscrResult::NoDiscriminant,
+        &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => DiscrResult::Value(
+            enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val,
         ),
         &Variants::Multiple {
-            tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant },
+            tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, untagged_variant },
             tag,
             ..
         } => {
-            if variant_index == dataful_variant {
-                None
+            if variant_index == untagged_variant {
+                let valid_range = enum_type_and_layout
+                    .for_variant(cx, variant_index)
+                    .largest_niche
+                    .as_ref()
+                    .unwrap()
+                    .valid_range;
+
+                let min = valid_range.start.min(valid_range.end);
+                let min = tag.size(cx).truncate(min);
+
+                let max = valid_range.start.max(valid_range.end);
+                let max = tag.size(cx).truncate(max);
+
+                DiscrResult::Range(min, max)
             } else {
                 let value = (variant_index.as_u32() as u128)
                     .wrapping_sub(niche_variants.start().as_u32() as u128)
                     .wrapping_add(niche_start);
                 let value = tag.size(cx).truncate(value);
-                // NOTE(eddyb) do *NOT* remove this assert, until
-                // we pass the full 128-bit value to LLVM, otherwise
-                // truncation will be silent and remain undetected.
-                assert_eq!(value as u64 as u128, value);
-                Some(value as u64)
+                DiscrResult::Value(value)
             }
         }
     }
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
index f1935e0ec..becbccc43 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
@@ -88,7 +88,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
                     variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()),
                     variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node(
                         cx,
-                        enum_type,
+                        enum_type_and_layout,
                         enum_type_di_node,
                         variant_index,
                         enum_adt_def.variant(variant_index),
@@ -378,7 +378,7 @@ fn build_discr_member_di_node<'ll, 'tcx>(
 ///
 /// The DW_AT_discr_value is optional, and is omitted if
 ///   - This is the only variant of a univariant enum (i.e. their is no discriminant)
-///   - This is the "dataful" variant of a niche-layout enum
+///   - This is the "untagged" variant of a niche-layout enum
 ///     (where only the other variants are identified by a single value)
 ///
 /// There is only ever a single member, the type of which is a struct that describes the
@@ -413,7 +413,13 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>(
             enum_type_and_layout.size.bits(),
             enum_type_and_layout.align.abi.bits() as u32,
             Size::ZERO.bits(),
-            discr_value.map(|v| cx.const_u64(v)),
+            discr_value.opt_single_val().map(|value| {
+                // NOTE(eddyb) do *NOT* remove this assert, until
+                // we pass the full 128-bit value to LLVM, otherwise
+                // truncation will be silent and remain undetected.
+                assert_eq!(value as u64 as u128, value);
+                cx.const_u64(value as u64)
+            }),
             DIFlags::FlagZero,
             variant_member_info.variant_struct_type_di_node,
         )
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs
index ce2f419c4..e30622cbd 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs
@@ -47,6 +47,8 @@ pub(super) enum UniqueTypeId<'tcx> {
     VariantPart(Ty<'tcx>, private::HiddenZst),
     /// The ID for the artificial struct type describing a single enum variant.
     VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst),
+    /// The ID for the additional wrapper struct type describing an enum variant in CPP-like mode.
+    VariantStructTypeCppLikeWrapper(Ty<'tcx>, VariantIdx, private::HiddenZst),
     /// The ID of the artificial type we create for VTables.
     VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, private::HiddenZst),
 }
@@ -71,6 +73,15 @@ impl<'tcx> UniqueTypeId<'tcx> {
         UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst)
     }
 
+    pub fn for_enum_variant_struct_type_wrapper(
+        tcx: TyCtxt<'tcx>,
+        enum_ty: Ty<'tcx>,
+        variant_idx: VariantIdx,
+    ) -> Self {
+        debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty));
+        UniqueTypeId::VariantStructTypeCppLikeWrapper(enum_ty, variant_idx, private::HiddenZst)
+    }
+
     pub fn for_vtable_ty(
         tcx: TyCtxt<'tcx>,
         self_type: Ty<'tcx>,
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index cf591295b..b23fe3fc9 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -39,7 +39,6 @@ use smallvec::SmallVec;
 use std::cell::OnceCell;
 use std::cell::RefCell;
 use std::iter;
-use tracing::debug;
 
 mod create_scope_map;
 pub mod gdb;
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
index 8f2436739..a40cfc8b2 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
@@ -6,7 +6,7 @@ use super::CodegenUnitDebugContext;
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::layout::{HasParamEnv, LayoutOf};
 use rustc_middle::ty::{self, DefIdTree, Ty};
-use tracing::trace;
+use trace;
 
 use crate::common::CodegenCx;
 use crate::llvm;
diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs
index fa0ecd18f..0f663a267 100644
--- a/compiler/rustc_codegen_llvm/src/declare.rs
+++ b/compiler/rustc_codegen_llvm/src/declare.rs
@@ -22,7 +22,6 @@ use rustc_codegen_ssa::traits::TypeMembershipMethods;
 use rustc_middle::ty::Ty;
 use rustc_symbol_mangling::typeid::typeid_for_fnabi;
 use smallvec::SmallVec;
-use tracing::debug;
 
 /// Declare a function.
 ///
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 9f3647492..a640de42a 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -71,6 +71,7 @@ fn get_simple_intrinsic<'ll>(
         sym::nearbyintf64 => "llvm.nearbyint.f64",
         sym::roundf32 => "llvm.round.f32",
         sym::roundf64 => "llvm.round.f64",
+        sym::ptr_mask => "llvm.ptrmask",
         _ => return None,
     };
     Some(cx.get_intrinsic(llvm_name))
@@ -161,7 +162,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             sym::volatile_load | sym::unaligned_volatile_load => {
                 let tp_ty = substs.type_at(0);
                 let ptr = args[0].immediate();
-                let load = if let PassMode::Cast(ty) = fn_abi.ret.mode {
+                let load = if let PassMode::Cast(ty, _) = &fn_abi.ret.mode {
                     let llty = ty.llvm_type(self);
                     let ptr = self.pointercast(ptr, self.type_ptr_to(llty));
                     self.volatile_load(llty, ptr)
@@ -374,7 +375,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         };
 
         if !fn_abi.ret.is_ignore() {
-            if let PassMode::Cast(ty) = fn_abi.ret.mode {
+            if let PassMode::Cast(ty, _) = &fn_abi.ret.mode {
                 let ptr_llty = self.type_ptr_to(ty.llvm_type(self));
                 let ptr = self.pointercast(result.llval, ptr_llty);
                 self.store(llval, ptr, result.align);
@@ -1704,6 +1705,97 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#,
     bitwise_red!(simd_reduce_all: vector_reduce_and, true);
     bitwise_red!(simd_reduce_any: vector_reduce_or, true);
 
+    if name == sym::simd_cast_ptr {
+        require_simd!(ret_ty, "return");
+        let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
+        require!(
+            in_len == out_len,
+            "expected return type with length {} (same as input type `{}`), \
+                  found `{}` with length {}",
+            in_len,
+            in_ty,
+            ret_ty,
+            out_len
+        );
+
+        match in_elem.kind() {
+            ty::RawPtr(p) => {
+                let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| {
+                    bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty)
+                });
+                assert!(!check_sized); // we are in codegen, so we shouldn't see these types
+                require!(metadata.is_unit(), "cannot cast fat pointer `{}`", in_elem)
+            }
+            _ => return_error!("expected pointer, got `{}`", in_elem),
+        }
+        match out_elem.kind() {
+            ty::RawPtr(p) => {
+                let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| {
+                    bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty)
+                });
+                assert!(!check_sized); // we are in codegen, so we shouldn't see these types
+                require!(metadata.is_unit(), "cannot cast to fat pointer `{}`", out_elem)
+            }
+            _ => return_error!("expected pointer, got `{}`", out_elem),
+        }
+
+        if in_elem == out_elem {
+            return Ok(args[0].immediate());
+        } else {
+            return Ok(bx.pointercast(args[0].immediate(), llret_ty));
+        }
+    }
+
+    if name == sym::simd_expose_addr {
+        require_simd!(ret_ty, "return");
+        let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
+        require!(
+            in_len == out_len,
+            "expected return type with length {} (same as input type `{}`), \
+                  found `{}` with length {}",
+            in_len,
+            in_ty,
+            ret_ty,
+            out_len
+        );
+
+        match in_elem.kind() {
+            ty::RawPtr(_) => {}
+            _ => return_error!("expected pointer, got `{}`", in_elem),
+        }
+        match out_elem.kind() {
+            ty::Uint(ty::UintTy::Usize) => {}
+            _ => return_error!("expected `usize`, got `{}`", out_elem),
+        }
+
+        return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
+    }
+
+    if name == sym::simd_from_exposed_addr {
+        require_simd!(ret_ty, "return");
+        let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
+        require!(
+            in_len == out_len,
+            "expected return type with length {} (same as input type `{}`), \
+                  found `{}` with length {}",
+            in_len,
+            in_ty,
+            ret_ty,
+            out_len
+        );
+
+        match in_elem.kind() {
+            ty::Uint(ty::UintTy::Usize) => {}
+            _ => return_error!("expected `usize`, got `{}`", in_elem),
+        }
+        match out_elem.kind() {
+            ty::RawPtr(_) => {}
+            _ => return_error!("expected pointer, got `{}`", out_elem),
+        }
+
+        return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
+    }
+
     if name == sym::simd_cast || name == sym::simd_as {
         require_simd!(ret_ty, "return");
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index 636d689a3..42c65e04e 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -7,7 +7,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(hash_raw_entry)]
 #![feature(let_chains)]
-#![feature(let_else)]
+#![cfg_attr(bootstrap, feature(let_else))]
 #![feature(extern_types)]
 #![feature(once_cell)]
 #![feature(iter_intersperse)]
@@ -16,6 +16,8 @@
 
 #[macro_use]
 extern crate rustc_macros;
+#[macro_use]
+extern crate tracing;
 
 use back::write::{create_informational_target_machine, create_target_machine};
 
diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs
index 64db4f746..7d9489702 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs
@@ -83,17 +83,6 @@ impl<'a> Child<'a> {
             }
         }
     }
-
-    pub fn data(&self) -> &'a [u8] {
-        unsafe {
-            let mut data_len = 0;
-            let data_ptr = super::LLVMRustArchiveChildData(self.raw, &mut data_len);
-            if data_ptr.is_null() {
-                panic!("failed to read data from archive child");
-            }
-            slice::from_raw_parts(data_ptr as *const u8, data_len as usize)
-        }
-    }
 }
 
 impl<'a> Drop for Child<'a> {
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 3139f93bf..ce27dc5a5 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -1096,7 +1096,7 @@ extern "C" {
     pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value;
     pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value;
     pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value;
-    pub fn LLVMConstIntGetZExtValue(ConstantVal: &ConstantInt) -> c_ulonglong;
+    pub fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool;
     pub fn LLVMRustConstInt128Get(
         ConstantVal: &ConstantInt,
         SExt: bool,
@@ -2079,6 +2079,19 @@ extern "C" {
         Ty: &'a DIType,
     ) -> &'a DIType;
 
+    pub fn LLVMRustDIBuilderCreateStaticMemberType<'a>(
+        Builder: &DIBuilder<'a>,
+        Scope: &'a DIDescriptor,
+        Name: *const c_char,
+        NameLen: size_t,
+        File: &'a DIFile,
+        LineNo: c_uint,
+        Ty: &'a DIType,
+        Flags: DIFlags,
+        val: Option<&'a Value>,
+        AlignInBits: u32,
+    ) -> &'a DIDerivedType;
+
     pub fn LLVMRustDIBuilderCreateLexicalBlock<'a>(
         Builder: &DIBuilder<'a>,
         Scope: &'a DIScope,
@@ -2347,6 +2360,7 @@ extern "C" {
         PGOGenPath: *const c_char,
         PGOUsePath: *const c_char,
         InstrumentCoverage: bool,
+        InstrProfileOutput: *const c_char,
         InstrumentGCOV: bool,
         PGOSampleUsePath: *const c_char,
         DebugInfoForProfiling: bool,
@@ -2375,7 +2389,6 @@ extern "C" {
         AIR: &ArchiveIterator<'a>,
     ) -> Option<&'a mut ArchiveChild<'a>>;
     pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char;
-    pub fn LLVMRustArchiveChildData(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char;
     pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>);
     pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>);
     pub fn LLVMRustDestroyArchive(AR: &'static mut Archive);
@@ -2410,12 +2423,6 @@ extern "C" {
         cookie_out: &mut c_uint,
     ) -> &'a SMDiagnostic;
 
-    pub fn LLVMRustSetInlineAsmDiagnosticHandler(
-        C: &Context,
-        H: InlineAsmDiagHandlerTy,
-        CX: *mut c_void,
-    );
-
     #[allow(improper_ctypes)]
     pub fn LLVMRustUnpackSMDiagnostic(
         d: &SMDiagnostic,
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index a0a640473..60707a1c3 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -15,7 +15,6 @@ use rustc_span::symbol::Symbol;
 use rustc_target::spec::{MergeFunctions, PanicStrategy};
 use smallvec::{smallvec, SmallVec};
 use std::ffi::{CStr, CString};
-use tracing::debug;
 
 use std::mem;
 use std::path::Path;
@@ -92,16 +91,6 @@ unsafe fn configure_llvm(sess: &Session) {
             add("-generate-arange-section", false);
         }
 
-        // Disable the machine outliner by default in LLVM versions 11 and LLVM
-        // version 12, where it leads to miscompilation.
-        //
-        // Ref:
-        // - https://github.com/rust-lang/rust/issues/85351
-        // - https://reviews.llvm.org/D103167
-        if llvm_util::get_version() < (13, 0, 0) {
-            add("-enable-machine-outliner=never", false);
-        }
-
         match sess.opts.unstable_opts.merge_functions.unwrap_or(sess.target.merge_functions) {
             MergeFunctions::Disabled | MergeFunctions::Trampolines => {}
             MergeFunctions::Aliases => {
@@ -165,6 +154,10 @@ pub fn time_trace_profiler_finish(file_name: &Path) {
 //
 // To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def
 // where the * matches the architecture's name
+//
+// For targets not present in the above location, see llvm-project/llvm/lib/Target/{ARCH}/*.td
+// where `{ARCH}` is the architecture name. Look for instances of `SubtargetFeature`.
+//
 // Beware to not use the llvm github project for this, but check the git submodule
 // found in src/llvm-project
 // Though note that Rust can also be build with an external precompiled version of LLVM
@@ -440,6 +433,8 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str
             .features
             .split(',')
             .filter(|v| !v.is_empty() && backend_feature_name(v).is_some())
+            // Drop +atomics-32 feature introduced in LLVM 15.
+            .filter(|v| *v != "+atomics-32" || get_version() >= (15, 0, 0))
             .map(String::from),
     );
 
diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs
index 6e9428485..1eceb7f5c 100644
--- a/compiler/rustc_codegen_llvm/src/mono_item.rs
+++ b/compiler/rustc_codegen_llvm/src/mono_item.rs
@@ -11,7 +11,6 @@ use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
 use rustc_middle::ty::{self, Instance, TypeVisitable};
 use rustc_session::config::CrateType;
 use rustc_target::spec::RelocModel;
-use tracing::debug;
 
 impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> {
     fn predefine_static(
diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs
index 9f0e6c80b..dc1165835 100644
--- a/compiler/rustc_codegen_llvm/src/type_of.rs
+++ b/compiler/rustc_codegen_llvm/src/type_of.rs
@@ -11,7 +11,6 @@ use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape};
 use rustc_target::abi::{Int, Pointer, F32, F64};
 use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants};
 use smallvec::{smallvec, SmallVec};
-use tracing::debug;
 
 use std::fmt::Write;