Merging upstream version 1.70.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

890 files changed, 30006 insertions, 20286 deletions

diff --git a/compiler/rustc/src/main.rs b/compiler/rustc/src/main.rs
index e21c9b660..434b978ae 100644
--- a/compiler/rustc/src/main.rs
+++ b/compiler/rustc/src/main.rs
@@ -24,6 +24,15 @@
 // The two crates we link to here, `std` and `rustc_driver`, are both dynamic
 // libraries. So we must reference jemalloc symbols one way or another, because
 // this file is the only object code in the rustc executable.
+//
+// NOTE: if you are reading this comment because you want to set a custom `global_allocator` for
+// benchmarking, consider using the benchmarks in the `rustc-perf` collector suite instead:
+// https://github.com/rust-lang/rustc-perf/blob/master/collector/README.md#profiling
+//
+// NOTE: if you are reading this comment because you want to replace jemalloc with another allocator
+// to compare their performance, see
+// https://github.com/rust-lang/rust/commit/b90cfc887c31c3e7a9e6d462e2464db1fe506175#diff-43914724af6e464c1da2171e4a9b6c7e607d5bc1203fa95c0ab85be4122605ef
+// for an example of how to do so.
 
 #[unix_sigpipe = "sig_dfl"]
 fn main() {
diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs
index 54858b520..c863acde7 100644
--- a/compiler/rustc_abi/src/layout.rs
+++ b/compiler/rustc_abi/src/layout.rs
@@ -8,19 +8,6 @@ use rand_xoshiro::Xoshiro128StarStar;
 
 use tracing::debug;
 
-// Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`.
-// This is used to go between `memory_index` (source field order to memory order)
-// and `inverse_memory_index` (memory order to source field order).
-// See also `FieldsShape::Arbitrary::memory_index` for more details.
-// FIXME(eddyb) build a better abstraction for permutations, if possible.
-fn invert_mapping(map: &[u32]) -> Vec<u32> {
-    let mut inverse = vec![0; map.len()];
-    for i in 0..map.len() {
-        inverse[map[i] as usize] = i as u32;
-    }
-    inverse
-}
-
 pub trait LayoutCalculator {
     type TargetDataLayoutRef: Borrow<TargetDataLayout>;
 
@@ -43,10 +30,10 @@ pub trait LayoutCalculator {
             .max_by_key(|niche| niche.available(dl));
 
         LayoutS {
-            variants: Variants::Single { index: VariantIdx::new(0) },
+            variants: Variants::Single { index: FIRST_VARIANT },
             fields: FieldsShape::Arbitrary {
-                offsets: vec![Size::ZERO, b_offset],
-                memory_index: vec![0, 1],
+                offsets: [Size::ZERO, b_offset].into(),
+                memory_index: [0, 1].into(),
             },
             abi: Abi::ScalarPair(a, b),
             largest_niche,
@@ -58,18 +45,18 @@ pub trait LayoutCalculator {
     fn univariant(
         &self,
         dl: &TargetDataLayout,
-        fields: &[Layout<'_>],
+        fields: &IndexSlice<FieldIdx, Layout<'_>>,
         repr: &ReprOptions,
         kind: StructKind,
     ) -> Option<LayoutS> {
         let pack = repr.pack;
         let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
-        let mut inverse_memory_index: Vec<u32> = (0..fields.len() as u32).collect();
+        let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect();
         let optimize = !repr.inhibit_struct_field_reordering_opt();
         if optimize {
             let end =
                 if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
-            let optimizing = &mut inverse_memory_index[..end];
+            let optimizing = &mut inverse_memory_index.raw[..end];
             let effective_field_align = |layout: Layout<'_>| {
                 if let Some(pack) = pack {
                     // return the packed alignment in bytes
@@ -105,7 +92,7 @@ pub trait LayoutCalculator {
                             // Place ZSTs first to avoid "interesting offsets",
                             // especially with only one or two non-ZST fields.
                             // Then place largest alignments first, largest niches within an alignment group last
-                            let f = fields[x as usize];
+                            let f = fields[x];
                             let niche_size = f.largest_niche().map_or(0, |n| n.available(dl));
                             (!f.0.is_zst(), cmp::Reverse(effective_field_align(f)), niche_size)
                         });
@@ -117,7 +104,7 @@ pub trait LayoutCalculator {
                         // And put the largest niche in an alignment group at the end
                         // so it can be used as discriminant in jagged enums
                         optimizing.sort_by_key(|&x| {
-                            let f = fields[x as usize];
+                            let f = fields[x];
                             let niche_size = f.largest_niche().map_or(0, |n| n.available(dl));
                             (effective_field_align(f), niche_size)
                         });
@@ -135,7 +122,7 @@ pub trait LayoutCalculator {
         // At the bottom of this function, we invert `inverse_memory_index` to
         // produce `memory_index` (see `invert_mapping`).
         let mut sized = true;
-        let mut offsets = vec![Size::ZERO; fields.len()];
+        let mut offsets = IndexVec::from_elem(Size::ZERO, &fields);
         let mut offset = Size::ZERO;
         let mut largest_niche = None;
         let mut largest_niche_available = 0;
@@ -146,7 +133,7 @@ pub trait LayoutCalculator {
             offset = prefix_size.align_to(prefix_align);
         }
         for &i in &inverse_memory_index {
-            let field = &fields[i as usize];
+            let field = &fields[i];
             if !sized {
                 self.delay_bug(&format!(
                     "univariant: field #{} comes after unsized field",
@@ -168,7 +155,7 @@ pub trait LayoutCalculator {
             align = align.max(field_align);
 
             debug!("univariant offset: {:?} field: {:#?}", offset, field);
-            offsets[i as usize] = offset;
+            offsets[i] = offset;
 
             if let Some(mut niche) = field.largest_niche() {
                 let available = niche.available(dl);
@@ -192,14 +179,18 @@ pub trait LayoutCalculator {
         // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
         // Field 5 would be the first element, so memory_index is i:
         // Note: if we didn't optimize, it's already right.
-        let memory_index =
-            if optimize { invert_mapping(&inverse_memory_index) } else { inverse_memory_index };
+        let memory_index = if optimize {
+            inverse_memory_index.invert_bijective_mapping()
+        } else {
+            debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
+            inverse_memory_index.into_iter().map(FieldIdx::as_u32).collect()
+        };
         let size = min_size.align_to(align.abi);
         let mut abi = Abi::Aggregate { sized };
         // Unpack newtype ABIs and find scalar pairs.
         if sized && size.bytes() > 0 {
             // All other fields must be ZSTs.
-            let mut non_zst_fields = fields.iter().enumerate().filter(|&(_, f)| !f.0.is_zst());
+            let mut non_zst_fields = fields.iter_enumerated().filter(|&(_, f)| !f.0.is_zst());
 
             match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) {
                 // We have exactly one non-ZST field.
@@ -238,13 +229,13 @@ pub trait LayoutCalculator {
                             let pair = self.scalar_pair(a, b);
                             let pair_offsets = match pair.fields {
                                 FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
-                                    assert_eq!(memory_index, &[0, 1]);
+                                    assert_eq!(memory_index.raw, [0, 1]);
                                     offsets
                                 }
                                 _ => panic!(),
                             };
-                            if offsets[i] == pair_offsets[0]
-                                && offsets[j] == pair_offsets[1]
+                            if offsets[i] == pair_offsets[FieldIdx::from_usize(0)]
+                                && offsets[j] == pair_offsets[FieldIdx::from_usize(1)]
                                 && align == pair.align
                                 && size == pair.size
                             {
@@ -264,7 +255,7 @@ pub trait LayoutCalculator {
             abi = Abi::Uninhabited;
         }
         Some(LayoutS {
-            variants: Variants::Single { index: VariantIdx::new(0) },
+            variants: Variants::Single { index: FIRST_VARIANT },
             fields: FieldsShape::Arbitrary { offsets, memory_index },
             abi,
             largest_niche,
@@ -277,7 +268,7 @@ pub trait LayoutCalculator {
         let dl = self.current_data_layout();
         let dl = dl.borrow();
         LayoutS {
-            variants: Variants::Single { index: VariantIdx::new(0) },
+            variants: Variants::Single { index: FIRST_VARIANT },
             fields: FieldsShape::Primitive,
             abi: Abi::Uninhabited,
             largest_niche: None,
@@ -289,7 +280,7 @@ pub trait LayoutCalculator {
     fn layout_of_struct_or_enum(
         &self,
         repr: &ReprOptions,
-        variants: &IndexVec<VariantIdx, Vec<Layout<'_>>>,
+        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, Layout<'_>>>,
         is_enum: bool,
         is_unsafe_cell: bool,
         scalar_valid_range: (Bound<u128>, Bound<u128>),
@@ -312,7 +303,7 @@ pub trait LayoutCalculator {
         // but *not* an encoding of the discriminant (e.g., a tag value).
         // See issue #49298 for more details on the need to leave space
         // for non-ZST uninhabited data (mostly partial initialization).
-        let absent = |fields: &[Layout<'_>]| {
+        let absent = |fields: &IndexSlice<FieldIdx, Layout<'_>>| {
             let uninhabited = fields.iter().any(|f| f.abi().is_uninhabited());
             let is_zst = fields.iter().all(|f| f.0.is_zst());
             uninhabited && is_zst
@@ -331,7 +322,7 @@ pub trait LayoutCalculator {
             }
             // If it's a struct, still compute a layout so that we can still compute the
             // field offsets.
-            None => VariantIdx::new(0),
+            None => FIRST_VARIANT,
         };
 
         let is_struct = !is_enum ||
@@ -467,7 +458,7 @@ pub trait LayoutCalculator {
                 .max_by_key(|(_i, layout)| layout.size.bytes())
                 .map(|(i, _layout)| i)?;
 
-            let all_indices = (0..=variants.len() - 1).map(VariantIdx::new);
+            let all_indices = variants.indices();
             let needs_disc =
                 |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]);
             let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap().index()
@@ -510,7 +501,7 @@ pub trait LayoutCalculator {
                 // It'll fit, but we need to make some adjustments.
                 match layout.fields {
                     FieldsShape::Arbitrary { ref mut offsets, .. } => {
-                        for (j, offset) in offsets.iter_mut().enumerate() {
+                        for (j, offset) in offsets.iter_enumerated_mut() {
                             if !variants[i][j].0.is_zst() {
                                 *offset += this_offset;
                             }
@@ -577,8 +568,8 @@ pub trait LayoutCalculator {
                     variants: IndexVec::new(),
                 },
                 fields: FieldsShape::Arbitrary {
-                    offsets: vec![niche_offset],
-                    memory_index: vec![0],
+                    offsets: [niche_offset].into(),
+                    memory_index: [0].into(),
                 },
                 abi,
                 largest_niche,
@@ -651,7 +642,8 @@ pub trait LayoutCalculator {
                 st.variants = Variants::Single { index: i };
                 // Find the first field we can't move later
                 // to make room for a larger discriminant.
-                for field in st.fields.index_by_increasing_offset().map(|j| &field_layouts[j]) {
+                for field_idx in st.fields.index_by_increasing_offset() {
+                    let field = &field_layouts[FieldIdx::from_usize(field_idx)];
                     if !field.0.is_zst() || field.align().abi.bytes() != 1 {
                         start_align = start_align.min(field.align().abi);
                         break;
@@ -802,13 +794,13 @@ pub trait LayoutCalculator {
                 let pair = self.scalar_pair(tag, prim_scalar);
                 let pair_offsets = match pair.fields {
                     FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
-                        assert_eq!(memory_index, &[0, 1]);
+                        assert_eq!(memory_index.raw, [0, 1]);
                         offsets
                     }
                     _ => panic!(),
                 };
-                if pair_offsets[0] == Size::ZERO
-                    && pair_offsets[1] == *offset
+                if pair_offsets[FieldIdx::from_u32(0)] == Size::ZERO
+                    && pair_offsets[FieldIdx::from_u32(1)] == *offset
                     && align == pair.align
                     && size == pair.size
                 {
@@ -844,7 +836,10 @@ pub trait LayoutCalculator {
                 tag_field: 0,
                 variants: IndexVec::new(),
             },
-            fields: FieldsShape::Arbitrary { offsets: vec![Size::ZERO], memory_index: vec![0] },
+            fields: FieldsShape::Arbitrary {
+                offsets: [Size::ZERO].into(),
+                memory_index: [0].into(),
+            },
             largest_niche,
             abi,
             align,
@@ -883,7 +878,7 @@ pub trait LayoutCalculator {
     fn layout_of_union(
         &self,
         repr: &ReprOptions,
-        variants: &IndexVec<VariantIdx, Vec<Layout<'_>>>,
+        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, Layout<'_>>>,
     ) -> Option<LayoutS> {
         let dl = self.current_data_layout();
         let dl = dl.borrow();
@@ -896,8 +891,8 @@ pub trait LayoutCalculator {
         let optimize = !repr.inhibit_union_abi_opt();
         let mut size = Size::ZERO;
         let mut abi = Abi::Aggregate { sized: true };
-        let index = VariantIdx::new(0);
-        for field in &variants[index] {
+        let only_variant = &variants[FIRST_VARIANT];
+        for field in only_variant {
             assert!(field.0.is_sized());
             align = align.max(field.align());
 
@@ -930,8 +925,8 @@ pub trait LayoutCalculator {
         }
 
         Some(LayoutS {
-            variants: Variants::Single { index },
-            fields: FieldsShape::Union(NonZeroUsize::new(variants[index].len())?),
+            variants: Variants::Single { index: FIRST_VARIANT },
+            fields: FieldsShape::Union(NonZeroUsize::new(only_variant.len())?),
             abi,
             largest_niche: None,
             align,
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 39574ca55..b0c0ee942 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -11,7 +11,7 @@ use bitflags::bitflags;
 use rustc_data_structures::intern::Interned;
 #[cfg(feature = "nightly")]
 use rustc_data_structures::stable_hasher::StableOrd;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{Idx, IndexSlice, IndexVec};
 #[cfg(feature = "nightly")]
 use rustc_macros::HashStable_Generic;
 #[cfg(feature = "nightly")]
@@ -1057,6 +1057,32 @@ impl Scalar {
     }
 }
 
+rustc_index::newtype_index! {
+    /// The *source-order* index of a field in a variant.
+    ///
+    /// This is how most code after type checking refers to fields, rather than
+    /// using names (as names have hygiene complications and more complex lookup).
+    ///
+    /// Particularly for `repr(Rust)` types, this may not be the same as *layout* order.
+    /// (It is for `repr(C)` `struct`s, however.)
+    ///
+    /// For example, in the following types,
+    /// ```rust
+    /// # enum Never {}
+    /// # #[repr(u16)]
+    /// enum Demo1 {
+    ///    Variant0 { a: Never, b: i32 } = 100,
+    ///    Variant1 { c: u8, d: u64 } = 10,
+    /// }
+    /// struct Demo2 { e: u8, f: u16, g: u8 }
+    /// ```
+    /// `b` is `FieldIdx(1)` in `VariantIdx(0)`,
+    /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and
+    /// `f` is `FieldIdx(1)` in `VariantIdx(0)`.
+    #[derive(HashStable_Generic)]
+    pub struct FieldIdx {}
+}
+
 /// Describes how the fields of a type are located in memory.
 #[derive(PartialEq, Eq, Hash, Clone, Debug)]
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
@@ -1082,7 +1108,7 @@ pub enum FieldsShape {
         /// ordered to match the source definition order.
         /// This vector does not go in increasing order.
         // FIXME(eddyb) use small vector optimization for the common case.
-        offsets: Vec<Size>,
+        offsets: IndexVec<FieldIdx, Size>,
 
         /// Maps source order field indices to memory order indices,
         /// depending on how the fields were reordered (if at all).
@@ -1096,7 +1122,7 @@ pub enum FieldsShape {
         ///
         // FIXME(eddyb) build a better abstraction for permutations, if possible.
         // FIXME(camlorn) also consider small vector optimization here.
-        memory_index: Vec<u32>,
+        memory_index: IndexVec<FieldIdx, u32>,
     },
 }
 
@@ -1131,7 +1157,7 @@ impl FieldsShape {
                 assert!(i < count);
                 stride * i
             }
-            FieldsShape::Arbitrary { ref offsets, .. } => offsets[i],
+            FieldsShape::Arbitrary { ref offsets, .. } => offsets[FieldIdx::from_usize(i)],
         }
     }
 
@@ -1142,28 +1168,27 @@ impl FieldsShape {
                 unreachable!("FieldsShape::memory_index: `Primitive`s have no fields")
             }
             FieldsShape::Union(_) | FieldsShape::Array { .. } => i,
-            FieldsShape::Arbitrary { ref memory_index, .. } => memory_index[i].try_into().unwrap(),
+            FieldsShape::Arbitrary { ref memory_index, .. } => {
+                memory_index[FieldIdx::from_usize(i)].try_into().unwrap()
+            }
         }
     }
 
     /// Gets source indices of the fields by increasing offsets.
     #[inline]
-    pub fn index_by_increasing_offset<'a>(&'a self) -> impl Iterator<Item = usize> + 'a {
+    pub fn index_by_increasing_offset(&self) -> impl Iterator<Item = usize> + '_ {
         let mut inverse_small = [0u8; 64];
-        let mut inverse_big = vec![];
+        let mut inverse_big = IndexVec::new();
         let use_small = self.count() <= inverse_small.len();
 
         // We have to write this logic twice in order to keep the array small.
         if let FieldsShape::Arbitrary { ref memory_index, .. } = *self {
             if use_small {
-                for i in 0..self.count() {
-                    inverse_small[memory_index[i] as usize] = i as u8;
+                for (field_idx, &mem_idx) in memory_index.iter_enumerated() {
+                    inverse_small[mem_idx as usize] = field_idx.as_u32() as u8;
                 }
             } else {
-                inverse_big = vec![0; self.count()];
-                for i in 0..self.count() {
-                    inverse_big[memory_index[i] as usize] = i as u32;
-                }
+                inverse_big = memory_index.invert_bijective_mapping();
             }
         }
 
@@ -1173,7 +1198,7 @@ impl FieldsShape {
                 if use_small {
                     inverse_small[i] as usize
                 } else {
-                    inverse_big[i] as usize
+                    inverse_big[i as u32].as_usize()
                 }
             }
         })
@@ -1380,8 +1405,21 @@ impl Niche {
 }
 
 rustc_index::newtype_index! {
+    /// The *source-order* index of a variant in a type.
+    ///
+    /// For enums, these are always `0..variant_count`, regardless of any
+    /// custom discriminants that may have been defined, and including any
+    /// variants that may end up uninhabited due to field types.  (Some of the
+    /// variants may not be present in a monomorphized ABI [`Variants`], but
+    /// those skipped variants are always counted when determining the *index*.)
+    ///
+    /// `struct`s, `tuples`, and `unions`s are considered to have a single variant
+    /// with variant index zero, aka [`FIRST_VARIANT`].
     #[derive(HashStable_Generic)]
-    pub struct VariantIdx {}
+    pub struct VariantIdx {
+        /// Equivalent to `VariantIdx(0)`.
+        const FIRST_VARIANT = 0;
+    }
 }
 
 #[derive(PartialEq, Eq, Hash, Clone)]
@@ -1422,7 +1460,7 @@ impl LayoutS {
         let size = scalar.size(cx);
         let align = scalar.align(cx);
         LayoutS {
-            variants: Variants::Single { index: VariantIdx::new(0) },
+            variants: Variants::Single { index: FIRST_VARIANT },
             fields: FieldsShape::Primitive,
             abi: Abi::Scalar(scalar),
             largest_niche,
@@ -1484,6 +1522,16 @@ impl<'a> Layout<'a> {
     pub fn size(self) -> Size {
         self.0.0.size
     }
+
+    /// Whether the layout is from a type that implements [`std::marker::PointerLike`].
+    ///
+    /// Currently, that means that the type is pointer-sized, pointer-aligned,
+    /// and has a scalar ABI.
+    pub fn is_pointer_like(self, data_layout: &TargetDataLayout) -> bool {
+        self.size() == data_layout.pointer_size
+            && self.align().abi == data_layout.pointer_align.abi
+            && matches!(self.abi(), Abi::Scalar(..))
+    }
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
diff --git a/compiler/rustc_arena/src/lib.rs b/compiler/rustc_arena/src/lib.rs
index 4fae5ef84..345e058e1 100644
--- a/compiler/rustc_arena/src/lib.rs
+++ b/compiler/rustc_arena/src/lib.rs
@@ -22,6 +22,7 @@
 #![feature(strict_provenance)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
+#![allow(clippy::mut_from_ref)] // Arena allocators are one of the places where this pattern is fine.
 
 use smallvec::SmallVec;
 
@@ -568,7 +569,9 @@ pub macro declare_arena([$($a:tt $name:ident: $ty:ty,)*]) {
     }
 
     pub trait ArenaAllocatable<'tcx, C = rustc_arena::IsNotCopy>: Sized {
+        #[allow(clippy::mut_from_ref)]
         fn allocate_on<'a>(self, arena: &'a Arena<'tcx>) -> &'a mut Self;
+        #[allow(clippy::mut_from_ref)]
         fn allocate_from_iter<'a>(
             arena: &'a Arena<'tcx>,
             iter: impl ::std::iter::IntoIterator<Item = Self>,
@@ -578,10 +581,12 @@ pub macro declare_arena([$($a:tt $name:ident: $ty:ty,)*]) {
     // Any type that impls `Copy` can be arena-allocated in the `DroplessArena`.
     impl<'tcx, T: Copy> ArenaAllocatable<'tcx, rustc_arena::IsCopy> for T {
         #[inline]
+        #[allow(clippy::mut_from_ref)]
         fn allocate_on<'a>(self, arena: &'a Arena<'tcx>) -> &'a mut Self {
             arena.dropless.alloc(self)
         }
         #[inline]
+        #[allow(clippy::mut_from_ref)]
         fn allocate_from_iter<'a>(
             arena: &'a Arena<'tcx>,
             iter: impl ::std::iter::IntoIterator<Item = Self>,
@@ -601,6 +606,7 @@ pub macro declare_arena([$($a:tt $name:ident: $ty:ty,)*]) {
             }
 
             #[inline]
+            #[allow(clippy::mut_from_ref)]
             fn allocate_from_iter<'a>(
                 arena: &'a Arena<'tcx>,
                 iter: impl ::std::iter::IntoIterator<Item = Self>,
@@ -616,12 +622,14 @@ pub macro declare_arena([$($a:tt $name:ident: $ty:ty,)*]) {
 
     impl<'tcx> Arena<'tcx> {
         #[inline]
+        #[allow(clippy::mut_from_ref)]
         pub fn alloc<T: ArenaAllocatable<'tcx, C>, C>(&self, value: T) -> &mut T {
             value.allocate_on(self)
         }
 
         // Any type that impls `Copy` can have slices be arena-allocated in the `DroplessArena`.
         #[inline]
+        #[allow(clippy::mut_from_ref)]
         pub fn alloc_slice<T: ::std::marker::Copy>(&self, value: &[T]) -> &mut [T] {
             if value.is_empty() {
                 return &mut [];
@@ -629,6 +637,7 @@ pub macro declare_arena([$($a:tt $name:ident: $ty:ty,)*]) {
             self.dropless.alloc_slice(value)
         }
 
+        #[allow(clippy::mut_from_ref)]
         pub fn alloc_from_iter<'a, T: ArenaAllocatable<'tcx, C>, C>(
             &'a self,
             iter: impl ::std::iter::IntoIterator<Item = T>,
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index 03c375c46..df1a71675 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -231,15 +231,15 @@ impl AngleBracketedArg {
     }
 }
 
-impl Into<Option<P<GenericArgs>>> for AngleBracketedArgs {
-    fn into(self) -> Option<P<GenericArgs>> {
-        Some(P(GenericArgs::AngleBracketed(self)))
+impl Into<P<GenericArgs>> for AngleBracketedArgs {
+    fn into(self) -> P<GenericArgs> {
+        P(GenericArgs::AngleBracketed(self))
     }
 }
 
-impl Into<Option<P<GenericArgs>>> for ParenthesizedArgs {
-    fn into(self) -> Option<P<GenericArgs>> {
-        Some(P(GenericArgs::Parenthesized(self)))
+impl Into<P<GenericArgs>> for ParenthesizedArgs {
+    fn into(self) -> P<GenericArgs> {
+        P(GenericArgs::Parenthesized(self))
     }
 }
 
@@ -1184,6 +1184,15 @@ impl Expr {
         expr
     }
 
+    pub fn peel_parens_and_refs(&self) -> &Expr {
+        let mut expr = self;
+        while let ExprKind::Paren(inner) | ExprKind::AddrOf(BorrowKind::Ref, _, inner) = &expr.kind
+        {
+            expr = inner;
+        }
+        expr
+    }
+
     /// Attempts to reparse as `Ty` (for diagnostic purposes).
     pub fn to_ty(&self) -> Option<P<Ty>> {
         let kind = match &self.kind {
@@ -1230,7 +1239,6 @@ impl Expr {
 
     pub fn precedence(&self) -> ExprPrecedence {
         match self.kind {
-            ExprKind::Box(_) => ExprPrecedence::Box,
             ExprKind::Array(_) => ExprPrecedence::Array,
             ExprKind::ConstBlock(_) => ExprPrecedence::ConstBlock,
             ExprKind::Call(..) => ExprPrecedence::Call,
@@ -1291,8 +1299,7 @@ impl Expr {
     /// To a first-order approximation, is this a pattern?
     pub fn is_approximately_pattern(&self) -> bool {
         match &self.peel_parens().kind {
-            ExprKind::Box(_)
-            | ExprKind::Array(_)
+            ExprKind::Array(_)
             | ExprKind::Call(_, _)
             | ExprKind::Tup(_)
             | ExprKind::Lit(_)
@@ -1363,8 +1370,6 @@ pub struct StructExpr {
 
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub enum ExprKind {
-    /// A `box x` expression.
-    Box(P<Expr>),
     /// An array (`[a, b, c, d]`)
     Array(ThinVec<P<Expr>>),
     /// Allow anonymous constants from an inline `const` block
@@ -1421,13 +1426,9 @@ pub enum ExprKind {
     Block(P<Block>, Option<Label>),
     /// An async block (`async move { ... }`).
     ///
-    /// The `NodeId` is the `NodeId` for the closure that results from
-    /// desugaring an async block, just like the NodeId field in the
-    /// `Async::Yes` variant. This is necessary in order to create a def for the
-    /// closure which can be used as a parent of any child defs. Defs
-    /// created during lowering cannot be made the parent of any other
-    /// preexisting defs.
-    Async(CaptureBy, NodeId, P<Block>),
+    /// The async block used to have a `NodeId`, which was removed in favor of
+    /// using the parent `NodeId` of the parent `Expr`.
+    Async(CaptureBy, P<Block>),
     /// An await expression (`my_future.await`).
     Await(P<Expr>),
 
@@ -2569,7 +2570,7 @@ pub enum AttrStyle {
 
 rustc_index::newtype_index! {
     #[custom_encodable]
-    #[debug_format = "AttrId({})]"]
+    #[debug_format = "AttrId({})"]
     pub struct AttrId {}
 }
 
@@ -2886,6 +2887,20 @@ pub struct Fn {
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
+pub struct StaticItem {
+    pub ty: P<Ty>,
+    pub mutability: Mutability,
+    pub expr: Option<P<Expr>>,
+}
+
+#[derive(Clone, Encodable, Decodable, Debug)]
+pub struct ConstItem {
+    pub defaultness: Defaultness,
+    pub ty: P<Ty>,
+    pub expr: Option<P<Expr>>,
+}
+
+#[derive(Clone, Encodable, Decodable, Debug)]
 pub enum ItemKind {
     /// An `extern crate` item, with the optional *original* crate name if the crate was renamed.
     ///
@@ -2898,11 +2913,11 @@ pub enum ItemKind {
     /// A static item (`static`).
     ///
     /// E.g., `static FOO: i32 = 42;` or `static FOO: &'static str = "bar";`.
-    Static(P<Ty>, Mutability, Option<P<Expr>>),
+    Static(Box<StaticItem>),
     /// A constant item (`const`).
     ///
     /// E.g., `const FOO: i32 = 42;`.
-    Const(Defaultness, P<Ty>, Option<P<Expr>>),
+    Const(Box<ConstItem>),
     /// A function declaration (`fn`).
     ///
     /// E.g., `fn foo(bar: usize) -> usize { .. }`.
@@ -3018,7 +3033,7 @@ pub type AssocItem = Item<AssocItemKind>;
 pub enum AssocItemKind {
     /// An associated constant, `const $ident: $ty $def?;` where `def ::= "=" $expr? ;`.
     /// If `def` is parsed, then the constant is provided, and otherwise required.
-    Const(Defaultness, P<Ty>, Option<P<Expr>>),
+    Const(Box<ConstItem>),
     /// An associated function.
     Fn(Box<Fn>),
     /// An associated type.
@@ -3030,7 +3045,7 @@ pub enum AssocItemKind {
 impl AssocItemKind {
     pub fn defaultness(&self) -> Defaultness {
         match *self {
-            Self::Const(defaultness, ..)
+            Self::Const(box ConstItem { defaultness, .. })
             | Self::Fn(box Fn { defaultness, .. })
             | Self::Type(box TyAlias { defaultness, .. }) => defaultness,
             Self::MacCall(..) => Defaultness::Final,
@@ -3041,7 +3056,7 @@ impl AssocItemKind {
 impl From<AssocItemKind> for ItemKind {
     fn from(assoc_item_kind: AssocItemKind) -> ItemKind {
         match assoc_item_kind {
-            AssocItemKind::Const(a, b, c) => ItemKind::Const(a, b, c),
+            AssocItemKind::Const(item) => ItemKind::Const(item),
             AssocItemKind::Fn(fn_kind) => ItemKind::Fn(fn_kind),
             AssocItemKind::Type(ty_alias_kind) => ItemKind::TyAlias(ty_alias_kind),
             AssocItemKind::MacCall(a) => ItemKind::MacCall(a),
@@ -3054,7 +3069,7 @@ impl TryFrom<ItemKind> for AssocItemKind {
 
     fn try_from(item_kind: ItemKind) -> Result<AssocItemKind, ItemKind> {
         Ok(match item_kind {
-            ItemKind::Const(a, b, c) => AssocItemKind::Const(a, b, c),
+            ItemKind::Const(item) => AssocItemKind::Const(item),
             ItemKind::Fn(fn_kind) => AssocItemKind::Fn(fn_kind),
             ItemKind::TyAlias(ty_kind) => AssocItemKind::Type(ty_kind),
             ItemKind::MacCall(a) => AssocItemKind::MacCall(a),
@@ -3079,7 +3094,9 @@ pub enum ForeignItemKind {
 impl From<ForeignItemKind> for ItemKind {
     fn from(foreign_item_kind: ForeignItemKind) -> ItemKind {
         match foreign_item_kind {
-            ForeignItemKind::Static(a, b, c) => ItemKind::Static(a, b, c),
+            ForeignItemKind::Static(a, b, c) => {
+                ItemKind::Static(StaticItem { ty: a, mutability: b, expr: c }.into())
+            }
             ForeignItemKind::Fn(fn_kind) => ItemKind::Fn(fn_kind),
             ForeignItemKind::TyAlias(ty_alias_kind) => ItemKind::TyAlias(ty_alias_kind),
             ForeignItemKind::MacCall(a) => ItemKind::MacCall(a),
@@ -3092,7 +3109,9 @@ impl TryFrom<ItemKind> for ForeignItemKind {
 
     fn try_from(item_kind: ItemKind) -> Result<ForeignItemKind, ItemKind> {
         Ok(match item_kind {
-            ItemKind::Static(a, b, c) => ForeignItemKind::Static(a, b, c),
+            ItemKind::Static(box StaticItem { ty: a, mutability: b, expr: c }) => {
+                ForeignItemKind::Static(a, b, c)
+            }
             ItemKind::Fn(fn_kind) => ForeignItemKind::Fn(fn_kind),
             ItemKind::TyAlias(ty_alias_kind) => ForeignItemKind::TyAlias(ty_alias_kind),
             ItemKind::MacCall(a) => ForeignItemKind::MacCall(a),
@@ -3109,8 +3128,8 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(AssocItem, 104);
-    static_assert_size!(AssocItemKind, 32);
+    static_assert_size!(AssocItem, 88);
+    static_assert_size!(AssocItemKind, 16);
     static_assert_size!(Attribute, 32);
     static_assert_size!(Block, 32);
     static_assert_size!(Expr, 72);
diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
index 2e83b3e62..c4771115c 100644
--- a/compiler/rustc_ast/src/attr/mod.rs
+++ b/compiler/rustc_ast/src/attr/mod.rs
@@ -180,6 +180,12 @@ impl Attribute {
         self.doc_str().map_or(false, |s| comments::may_have_doc_links(s.as_str()))
     }
 
+    pub fn is_proc_macro_attr(&self) -> bool {
+        [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive]
+            .iter()
+            .any(|kind| self.has_name(*kind))
+    }
+
     /// Extracts the MetaItem from inside this Attribute.
     pub fn meta(&self) -> Option<MetaItem> {
         match &self.kind {
@@ -627,6 +633,22 @@ pub fn mk_attr_name_value_str(
     mk_attr(g, style, path, args, span)
 }
 
+pub fn filter_by_name(attrs: &[Attribute], name: Symbol) -> impl Iterator<Item = &Attribute> {
+    attrs.iter().filter(move |attr| attr.has_name(name))
+}
+
+pub fn find_by_name(attrs: &[Attribute], name: Symbol) -> Option<&Attribute> {
+    filter_by_name(attrs, name).next()
+}
+
+pub fn first_attr_value_str_by_name(attrs: &[Attribute], name: Symbol) -> Option<Symbol> {
+    find_by_name(attrs, name).and_then(|attr| attr.value_str())
+}
+
+pub fn contains_name(attrs: &[Attribute], name: Symbol) -> bool {
+    find_by_name(attrs, name).is_some()
+}
+
 pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
     items.iter().any(|item| item.has_name(name))
 }
diff --git a/compiler/rustc_ast/src/format.rs b/compiler/rustc_ast/src/format.rs
index d021bea5e..699946f30 100644
--- a/compiler/rustc_ast/src/format.rs
+++ b/compiler/rustc_ast/src/format.rs
@@ -94,7 +94,7 @@ impl FormatArguments {
         }
         if !matches!(arg.kind, FormatArgumentKind::Captured(..)) {
             // This is an explicit argument.
-            // Make sure that all arguments so far are explcit.
+            // Make sure that all arguments so far are explicit.
             assert_eq!(
                 self.num_explicit_args,
                 self.arguments.len(),
@@ -131,8 +131,8 @@ impl FormatArguments {
         &self.arguments[..]
     }
 
-    pub fn all_args_mut(&mut self) -> &mut [FormatArgument] {
-        &mut self.arguments[..]
+    pub fn all_args_mut(&mut self) -> &mut Vec<FormatArgument> {
+        &mut self.arguments
     }
 }
 
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index 7dcb03b4c..694d31d8f 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -7,12 +7,12 @@
 //! a `MutVisitor` renaming item names in a module will miss all of those
 //! that are created by the expansion of a macro.
 
-use crate::ast::*;
 use crate::ptr::P;
 use crate::token::{self, Token};
 use crate::tokenstream::*;
+use crate::{ast::*, StaticItem};
 
-use rustc_data_structures::map_in_place::MapInPlace;
+use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
 use rustc_data_structures::sync::Lrc;
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::Ident;
@@ -1029,14 +1029,12 @@ pub fn noop_visit_item_kind<T: MutVisitor>(kind: &mut ItemKind, vis: &mut T) {
     match kind {
         ItemKind::ExternCrate(_orig_name) => {}
         ItemKind::Use(use_tree) => vis.visit_use_tree(use_tree),
-        ItemKind::Static(ty, _, expr) => {
+        ItemKind::Static(box StaticItem { ty, mutability: _, expr }) => {
             vis.visit_ty(ty);
             visit_opt(expr, |expr| vis.visit_expr(expr));
         }
-        ItemKind::Const(defaultness, ty, expr) => {
-            visit_defaultness(defaultness, vis);
-            vis.visit_ty(ty);
-            visit_opt(expr, |expr| vis.visit_expr(expr));
+        ItemKind::Const(item) => {
+            visit_const_item(item, vis);
         }
         ItemKind::Fn(box Fn { defaultness, generics, sig, body }) => {
             visit_defaultness(defaultness, vis);
@@ -1119,10 +1117,8 @@ pub fn noop_flat_map_assoc_item<T: MutVisitor>(
     visitor.visit_vis(vis);
     visit_attrs(attrs, visitor);
     match kind {
-        AssocItemKind::Const(defaultness, ty, expr) => {
-            visit_defaultness(defaultness, visitor);
-            visitor.visit_ty(ty);
-            visit_opt(expr, |expr| visitor.visit_expr(expr));
+        AssocItemKind::Const(item) => {
+            visit_const_item(item, visitor);
         }
         AssocItemKind::Fn(box Fn { defaultness, generics, sig, body }) => {
             visit_defaultness(defaultness, visitor);
@@ -1152,6 +1148,15 @@ pub fn noop_flat_map_assoc_item<T: MutVisitor>(
     smallvec![item]
 }
 
+fn visit_const_item<T: MutVisitor>(
+    ConstItem { defaultness, ty, expr }: &mut ConstItem,
+    visitor: &mut T,
+) {
+    visit_defaultness(defaultness, visitor);
+    visitor.visit_ty(ty);
+    visit_opt(expr, |expr| visitor.visit_expr(expr));
+}
+
 pub fn noop_visit_fn_header<T: MutVisitor>(header: &mut FnHeader, vis: &mut T) {
     let FnHeader { unsafety, asyncness, constness, ext: _ } = header;
     visit_constness(constness, vis);
@@ -1316,7 +1321,6 @@ pub fn noop_visit_expr<T: MutVisitor>(
     vis: &mut T,
 ) {
     match kind {
-        ExprKind::Box(expr) => vis.visit_expr(expr),
         ExprKind::Array(exprs) => visit_thin_exprs(exprs, vis),
         ExprKind::ConstBlock(anon_const) => {
             vis.visit_anon_const(anon_const);
@@ -1408,8 +1412,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
             vis.visit_block(blk);
             visit_opt(label, |label| vis.visit_label(label));
         }
-        ExprKind::Async(_capture_by, node_id, body) => {
-            vis.visit_id(node_id);
+        ExprKind::Async(_capture_by, body) => {
             vis.visit_block(body);
         }
         ExprKind::Await(expr) => vis.visit_expr(expr),
diff --git a/compiler/rustc_ast/src/util/classify.rs b/compiler/rustc_ast/src/util/classify.rs
index cdc244c12..607b77705 100644
--- a/compiler/rustc_ast/src/util/classify.rs
+++ b/compiler/rustc_ast/src/util/classify.rs
@@ -35,7 +35,6 @@ pub fn expr_trailing_brace(mut expr: &ast::Expr) -> Option<&ast::Expr> {
             | Assign(_, e, _)
             | AssignOp(_, _, e)
             | Binary(_, _, e)
-            | Box(e)
             | Break(_, Some(e))
             | Let(_, e, _)
             | Range(_, Some(e), _)
diff --git a/compiler/rustc_ast/src/util/parser.rs b/compiler/rustc_ast/src/util/parser.rs
index 3a0af04f9..3893875e9 100644
--- a/compiler/rustc_ast/src/util/parser.rs
+++ b/compiler/rustc_ast/src/util/parser.rs
@@ -259,7 +259,6 @@ pub enum ExprPrecedence {
     Assign,
     AssignOp,
 
-    Box,
     AddrOf,
     Let,
     Unary,
@@ -319,8 +318,7 @@ impl ExprPrecedence {
             ExprPrecedence::AssignOp => AssocOp::Assign.precedence() as i8,
 
             // Unary, prefix
-            ExprPrecedence::Box
-            | ExprPrecedence::AddrOf
+            ExprPrecedence::AddrOf
             // Here `let pats = expr` has `let pats =` as a "unary" prefix of `expr`.
             // However, this is not exactly right. When `let _ = a` is the LHS of a binop we
             // need parens sometimes. E.g. we can print `(let _ = a) && b` as `let _ = a && b`
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index bdb1879ec..ac9b321b7 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -13,7 +13,7 @@
 //! instance, a walker looking for item names in a module will miss all of
 //! those that are created by the expansion of a macro.
 
-use crate::ast::*;
+use crate::{ast::*, StaticItem};
 
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
@@ -305,8 +305,9 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) {
     match &item.kind {
         ItemKind::ExternCrate(_) => {}
         ItemKind::Use(use_tree) => visitor.visit_use_tree(use_tree, item.id, false),
-        ItemKind::Static(typ, _, expr) | ItemKind::Const(_, typ, expr) => {
-            visitor.visit_ty(typ);
+        ItemKind::Static(box StaticItem { ty, mutability: _, expr })
+        | ItemKind::Const(box ConstItem { ty, expr, .. }) => {
+            visitor.visit_ty(ty);
             walk_list!(visitor, visit_expr, expr);
         }
         ItemKind::Fn(box Fn { defaultness: _, generics, sig, body }) => {
@@ -673,7 +674,7 @@ pub fn walk_assoc_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a AssocItem,
     visitor.visit_ident(ident);
     walk_list!(visitor, visit_attribute, attrs);
     match kind {
-        AssocItemKind::Const(_, ty, expr) => {
+        AssocItemKind::Const(box ConstItem { ty, expr, .. }) => {
             visitor.visit_ty(ty);
             walk_list!(visitor, visit_expr, expr);
         }
@@ -772,7 +773,6 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
     walk_list!(visitor, visit_attribute, expression.attrs.iter());
 
     match &expression.kind {
-        ExprKind::Box(subexpression) => visitor.visit_expr(subexpression),
         ExprKind::Array(subexpressions) => {
             walk_list!(visitor, visit_expr, subexpressions);
         }
@@ -861,7 +861,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
             walk_list!(visitor, visit_label, opt_label);
             visitor.visit_block(block);
         }
-        ExprKind::Async(_, _, body) => {
+        ExprKind::Async(_, body) => {
             visitor.visit_block(body);
         }
         ExprKind::Await(expr) => visitor.visit_expr(expr),
diff --git a/compiler/rustc_ast_lowering/locales/en-US.ftl b/compiler/rustc_ast_lowering/messages.ftl
index 3ccd84398..21b2a3c22 100644
--- a/compiler/rustc_ast_lowering/locales/en-US.ftl
+++ b/compiler/rustc_ast_lowering/messages.ftl
@@ -139,3 +139,15 @@ ast_lowering_trait_fn_async =
     .label = `async` because of this
     .note = `async` trait functions are not currently supported
     .note2 = consider using the `async-trait` crate: https://crates.io/crates/async-trait
+
+ast_lowering_bad_return_type_notation_inputs =
+    argument types not allowed with return type notation
+    .suggestion = remove the input types
+
+ast_lowering_bad_return_type_notation_needs_dots =
+    return type notation arguments must be elided with `..`
+    .suggestion = add `..`
+
+ast_lowering_bad_return_type_notation_output =
+    return type not allowed with return type notation
+    .suggestion = remove the return type
diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs
index 5e6b6050b..3e9f9b436 100644
--- a/compiler/rustc_ast_lowering/src/errors.rs
+++ b/compiler/rustc_ast_lowering/src/errors.rs
@@ -137,7 +137,7 @@ pub struct AsyncNonMoveClosureNotSupported {
 
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_functional_record_update_destructuring_assignment)]
-pub struct FunctionalRecordUpdateDestructuringAssignemnt {
+pub struct FunctionalRecordUpdateDestructuringAssignment {
     #[primary_span]
     #[suggestion(code = "", applicability = "machine-applicable")]
     pub span: Span,
@@ -347,3 +347,19 @@ pub struct TraitFnAsync {
     #[label]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+pub enum BadReturnTypeNotation {
+    #[diag(ast_lowering_bad_return_type_notation_inputs)]
+    Inputs {
+        #[primary_span]
+        #[suggestion(code = "()", applicability = "maybe-incorrect")]
+        span: Span,
+    },
+    #[diag(ast_lowering_bad_return_type_notation_output)]
+    Output {
+        #[primary_span]
+        #[suggestion(code = "", applicability = "maybe-incorrect")]
+        span: Span,
+    },
+}
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index ffb30b1b3..1b1c4765b 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -1,6 +1,6 @@
 use super::errors::{
     AsyncGeneratorsNotSupported, AsyncNonMoveClosureNotSupported, AwaitOnlyInAsyncFnAndBlocks,
-    BaseExpressionDoubleDot, ClosureCannotBeStatic, FunctionalRecordUpdateDestructuringAssignemnt,
+    BaseExpressionDoubleDot, ClosureCannotBeStatic, FunctionalRecordUpdateDestructuringAssignment,
     GeneratorTooManyParameters, InclusiveRangeWithNoEnd, NotSupportedForLifetimeBinderAsyncClosure,
     UnderscoreExprLhsAssign,
 };
@@ -32,7 +32,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     pub(super) fn lower_expr_mut(&mut self, e: &Expr) -> hir::Expr<'hir> {
         ensure_sufficient_stack(|| {
             match &e.kind {
-                // Paranthesis expression does not have a HirId and is handled specially.
+                // Parenthesis expression does not have a HirId and is handled specially.
                 ExprKind::Paren(ex) => {
                     let mut ex = self.lower_expr_mut(ex);
                     // Include parens in span, but only if it is a super-span.
@@ -70,7 +70,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
             self.lower_attrs(hir_id, &e.attrs);
 
             let kind = match &e.kind {
-                ExprKind::Box(inner) => hir::ExprKind::Box(self.lower_expr(inner)),
                 ExprKind::Array(exprs) => hir::ExprKind::Array(self.lower_exprs(exprs)),
                 ExprKind::ConstBlock(anon_const) => {
                     let anon_const = self.lower_anon_const(anon_const);
@@ -174,10 +173,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     self.arena.alloc_from_iter(arms.iter().map(|x| self.lower_arm(x))),
                     hir::MatchSource::Normal,
                 ),
-                ExprKind::Async(capture_clause, closure_node_id, block) => self.make_async_expr(
+                ExprKind::Async(capture_clause, block) => self.make_async_expr(
                     *capture_clause,
-                    hir_id,
-                    *closure_node_id,
+                    e.id,
                     None,
                     e.span,
                     hir::AsyncGeneratorKind::Block,
@@ -316,7 +314,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 ),
                 ExprKind::Try(sub_expr) => self.lower_expr_try(e.span, sub_expr),
 
-                ExprKind::Paren(_) | ExprKind::ForLoop(..) => unreachable!("already handled"),
+                ExprKind::Paren(_) | ExprKind::ForLoop(..) => {
+                    unreachable!("already handled")
+                }
 
                 ExprKind::MacCall(_) => panic!("{:?} shouldn't exist here", e.span),
             };
@@ -434,7 +434,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         // `if let pat = val` or `if foo && let pat = val`, as we _do_ want `val` to live beyond the
         // condition in this case.
         //
-        // In order to mantain the drop behavior for the non `let` parts of the condition,
+        // In order to maintain the drop behavior for the non `let` parts of the condition,
         // we still wrap them in terminating scopes, e.g. `if foo && let pat = val` essentially
         // gets transformed into `if { let _t = foo; _t } && let pat = val`
         match &cond.kind {
@@ -578,14 +578,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
     /// This results in:
     ///
     /// ```text
-    /// std::future::identity_future(static move? |_task_context| -> <ret_ty> {
+    /// static move? |_task_context| -> <ret_ty> {
     ///     <body>
-    /// })
+    /// }
     /// ```
     pub(super) fn make_async_expr(
         &mut self,
         capture_clause: CaptureBy,
-        outer_hir_id: hir::HirId,
         closure_node_id: NodeId,
         ret_ty: Option<hir::FnRetTy<'hir>>,
         span: Span,
@@ -638,33 +637,36 @@ impl<'hir> LoweringContext<'_, 'hir> {
         });
 
         // `static |_task_context| -> <ret_ty> { body }`:
-        let generator_kind = {
-            let c = self.arena.alloc(hir::Closure {
-                def_id: self.local_def_id(closure_node_id),
-                binder: hir::ClosureBinder::Default,
-                capture_clause,
-                bound_generic_params: &[],
-                fn_decl,
-                body,
-                fn_decl_span: self.lower_span(span),
-                fn_arg_span: None,
-                movability: Some(hir::Movability::Static),
-                constness: hir::Constness::NotConst,
-            });
-
-            hir::ExprKind::Closure(c)
-        };
-
-        let hir_id = self.lower_node_id(closure_node_id);
-        let unstable_span =
-            self.mark_span_with_reason(DesugaringKind::Async, span, self.allow_gen_future.clone());
+        hir::ExprKind::Closure(self.arena.alloc(hir::Closure {
+            def_id: self.local_def_id(closure_node_id),
+            binder: hir::ClosureBinder::Default,
+            capture_clause,
+            bound_generic_params: &[],
+            fn_decl,
+            body,
+            fn_decl_span: self.lower_span(span),
+            fn_arg_span: None,
+            movability: Some(hir::Movability::Static),
+            constness: hir::Constness::NotConst,
+        }))
+    }
 
+    /// Forwards a possible `#[track_caller]` annotation from `outer_hir_id` to
+    /// `inner_hir_id` in case the `closure_track_caller` feature is enabled.
+    pub(super) fn maybe_forward_track_caller(
+        &mut self,
+        span: Span,
+        outer_hir_id: hir::HirId,
+        inner_hir_id: hir::HirId,
+    ) {
         if self.tcx.features().closure_track_caller
             && let Some(attrs) = self.attrs.get(&outer_hir_id.local_id)
             && attrs.into_iter().any(|attr| attr.has_name(sym::track_caller))
         {
+            let unstable_span =
+                self.mark_span_with_reason(DesugaringKind::Async, span, self.allow_gen_future.clone());
             self.lower_attrs(
-                hir_id,
+                inner_hir_id,
                 &[Attribute {
                     kind: AttrKind::Normal(ptr::P(NormalAttr {
                         item: AttrItem {
@@ -680,23 +682,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }],
             );
         }
-
-        let generator = hir::Expr { hir_id, kind: generator_kind, span: self.lower_span(span) };
-
-        // FIXME(swatinem):
-        // For some reason, the async block needs to flow through *any*
-        // call (like the identity function), as otherwise type and lifetime
-        // inference have a hard time figuring things out.
-        // Without this, we would get:
-        // E0720 in tests/ui/impl-trait/in-trait/default-body-with-rpit.rs
-        // E0700 in tests/ui/self/self_lifetime-async.rs
-
-        // `future::identity_future`:
-        let identity_future =
-            self.expr_lang_item_path(unstable_span, hir::LangItem::IdentityFuture, None);
-
-        // `future::identity_future(generator)`:
-        hir::ExprKind::Call(self.arena.alloc(identity_future), arena_vec![self; generator])
     }
 
     /// Desugar `<expr>.await` into:
@@ -1002,7 +987,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
 
             // Transform `async |x: u8| -> X { ... }` into
-            // `|x: u8| identity_future(|| -> X { ... })`.
+            // `|x: u8| || -> X { ... }`.
             let body_id = this.lower_fn_body(&outer_decl, |this| {
                 let async_ret_ty = if let FnRetTy::Ty(ty) = &decl.output {
                     let itctx = ImplTraitContext::Disallowed(ImplTraitPosition::AsyncBlock);
@@ -1013,14 +998,15 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
                 let async_body = this.make_async_expr(
                     capture_clause,
-                    closure_hir_id,
                     inner_closure_id,
                     async_ret_ty,
                     body.span,
                     hir::AsyncGeneratorKind::Closure,
                     |this| this.with_new_scopes(|this| this.lower_expr_mut(body)),
                 );
-                this.expr(fn_decl_span, async_body)
+                let hir_id = this.lower_node_id(inner_closure_id);
+                this.maybe_forward_track_caller(body.span, closure_hir_id, hir_id);
+                hir::Expr { hir_id, kind: async_body, span: this.lower_span(body.span) }
             });
             body_id
         });
@@ -1246,7 +1232,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 );
                 let fields_omitted = match &se.rest {
                     StructRest::Base(e) => {
-                        self.tcx.sess.emit_err(FunctionalRecordUpdateDestructuringAssignemnt {
+                        self.tcx.sess.emit_err(FunctionalRecordUpdateDestructuringAssignment {
                             span: e.span,
                         });
                         true
diff --git a/compiler/rustc_ast_lowering/src/format.rs b/compiler/rustc_ast_lowering/src/format.rs
index 4095e225a..c41bdc440 100644
--- a/compiler/rustc_ast_lowering/src/format.rs
+++ b/compiler/rustc_ast_lowering/src/format.rs
@@ -2,18 +2,177 @@ use super::LoweringContext;
 use rustc_ast as ast;
 use rustc_ast::visit::{self, Visitor};
 use rustc_ast::*;
-use rustc_data_structures::fx::FxIndexSet;
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir as hir;
 use rustc_span::{
     sym,
     symbol::{kw, Ident},
-    Span,
+    Span, Symbol,
 };
+use std::borrow::Cow;
 
 impl<'hir> LoweringContext<'_, 'hir> {
     pub(crate) fn lower_format_args(&mut self, sp: Span, fmt: &FormatArgs) -> hir::ExprKind<'hir> {
-        expand_format_args(self, sp, fmt)
+        // Never call the const constructor of `fmt::Arguments` if the
+        // format_args!() had any arguments _before_ flattening/inlining.
+        let allow_const = fmt.arguments.all_args().is_empty();
+        let mut fmt = Cow::Borrowed(fmt);
+        if self.tcx.sess.opts.unstable_opts.flatten_format_args {
+            fmt = flatten_format_args(fmt);
+            fmt = inline_literals(fmt);
+        }
+        expand_format_args(self, sp, &fmt, allow_const)
+    }
+}
+
+/// Flattens nested `format_args!()` into one.
+///
+/// Turns
+///
+/// `format_args!("a {} {} {}.", 1, format_args!("b{}!", 2), 3)`
+///
+/// into
+///
+/// `format_args!("a {} b{}! {}.", 1, 2, 3)`.
+fn flatten_format_args(mut fmt: Cow<'_, FormatArgs>) -> Cow<'_, FormatArgs> {
+    let mut i = 0;
+    while i < fmt.template.len() {
+        if let FormatArgsPiece::Placeholder(placeholder) = &fmt.template[i]
+            && let FormatTrait::Display | FormatTrait::Debug = &placeholder.format_trait
+            && let Ok(arg_index) = placeholder.argument.index
+            && let arg = fmt.arguments.all_args()[arg_index].expr.peel_parens_and_refs()
+            && let ExprKind::FormatArgs(_) = &arg.kind
+            // Check that this argument is not used by any other placeholders.
+            && fmt.template.iter().enumerate().all(|(j, p)|
+                i == j ||
+                !matches!(p, FormatArgsPiece::Placeholder(placeholder)
+                    if placeholder.argument.index == Ok(arg_index))
+            )
+        {
+            // Now we need to mutate the outer FormatArgs.
+            // If this is the first time, this clones the outer FormatArgs.
+            let fmt = fmt.to_mut();
+
+            // Take the inner FormatArgs out of the outer arguments, and
+            // replace it by the inner arguments. (We can't just put those at
+            // the end, because we need to preserve the order of evaluation.)
+
+            let args = fmt.arguments.all_args_mut();
+            let remaining_args = args.split_off(arg_index + 1);
+            let old_arg_offset = args.len();
+            let mut fmt2 = &mut args.pop().unwrap().expr; // The inner FormatArgs.
+            let fmt2 = loop { // Unwrap the Expr to get to the FormatArgs.
+                match &mut fmt2.kind {
+                    ExprKind::Paren(inner) | ExprKind::AddrOf(BorrowKind::Ref, _, inner) => fmt2 = inner,
+                    ExprKind::FormatArgs(fmt2) => break fmt2,
+                    _ => unreachable!(),
+                }
+            };
+
+            args.append(fmt2.arguments.all_args_mut());
+            let new_arg_offset = args.len();
+            args.extend(remaining_args);
+
+            // Correct the indexes that refer to the arguments after the newly inserted arguments.
+            for_all_argument_indexes(&mut fmt.template, |index| {
+                if *index >= old_arg_offset {
+                    *index -= old_arg_offset;
+                    *index += new_arg_offset;
+                }
+            });
+
+            // Now merge the placeholders:
+
+            let rest = fmt.template.split_off(i + 1);
+            fmt.template.pop(); // remove the placeholder for the nested fmt args.
+            // Insert the pieces from the nested format args, but correct any
+            // placeholders to point to the correct argument index.
+            for_all_argument_indexes(&mut fmt2.template, |index| *index += arg_index);
+            fmt.template.append(&mut fmt2.template);
+            fmt.template.extend(rest);
+
+            // Don't increment `i` here, so we recurse into the newly added pieces.
+        } else {
+            i += 1;
+        }
     }
+    fmt
+}
+
+/// Inline literals into the format string.
+///
+/// Turns
+///
+/// `format_args!("Hello, {}! {} {}", "World", 123, x)`
+///
+/// into
+///
+/// `format_args!("Hello, World! 123 {}", x)`.
+fn inline_literals(mut fmt: Cow<'_, FormatArgs>) -> Cow<'_, FormatArgs> {
+    let mut was_inlined = vec![false; fmt.arguments.all_args().len()];
+    let mut inlined_anything = false;
+
+    for i in 0..fmt.template.len() {
+        let FormatArgsPiece::Placeholder(placeholder) = &fmt.template[i] else { continue };
+        let Ok(arg_index) = placeholder.argument.index else { continue };
+
+        let mut literal = None;
+
+        if let FormatTrait::Display = placeholder.format_trait
+            && placeholder.format_options == Default::default()
+            && let arg = fmt.arguments.all_args()[arg_index].expr.peel_parens_and_refs()
+            && let ExprKind::Lit(lit) = arg.kind
+        {
+            if let token::LitKind::Str | token::LitKind::StrRaw(_) = lit.kind
+                && let Ok(LitKind::Str(s, _)) = LitKind::from_token_lit(lit)
+            {
+                literal = Some(s);
+            } else if let token::LitKind::Integer = lit.kind
+                && let Ok(LitKind::Int(n, _)) = LitKind::from_token_lit(lit)
+            {
+                literal = Some(Symbol::intern(&n.to_string()));
+            }
+        }
+
+        if let Some(literal) = literal {
+            // Now we need to mutate the outer FormatArgs.
+            // If this is the first time, this clones the outer FormatArgs.
+            let fmt = fmt.to_mut();
+            // Replace the placeholder with the literal.
+            fmt.template[i] = FormatArgsPiece::Literal(literal);
+            was_inlined[arg_index] = true;
+            inlined_anything = true;
+        }
+    }
+
+    // Remove the arguments that were inlined.
+    if inlined_anything {
+        let fmt = fmt.to_mut();
+
+        let mut remove = was_inlined;
+
+        // Don't remove anything that's still used.
+        for_all_argument_indexes(&mut fmt.template, |index| remove[*index] = false);
+
+        // Drop all the arguments that are marked for removal.
+        let mut remove_it = remove.iter();
+        fmt.arguments.all_args_mut().retain(|_| remove_it.next() != Some(&true));
+
+        // Calculate the mapping of old to new indexes for the remaining arguments.
+        let index_map: Vec<usize> = remove
+            .into_iter()
+            .scan(0, |i, remove| {
+                let mapped = *i;
+                *i += !remove as usize;
+                Some(mapped)
+            })
+            .collect();
+
+        // Correct the indexes that refer to arguments that have shifted position.
+        for_all_argument_indexes(&mut fmt.template, |index| *index = index_map[*index]);
+    }
+
+    fmt
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)]
@@ -79,7 +238,7 @@ fn make_count<'hir>(
     ctx: &mut LoweringContext<'_, 'hir>,
     sp: Span,
     count: &Option<FormatCount>,
-    argmap: &mut FxIndexSet<(usize, ArgumentType)>,
+    argmap: &mut FxIndexMap<(usize, ArgumentType), Option<Span>>,
 ) -> hir::Expr<'hir> {
     match count {
         Some(FormatCount::Literal(n)) => {
@@ -93,7 +252,7 @@ fn make_count<'hir>(
         }
         Some(FormatCount::Argument(arg)) => {
             if let Ok(arg_index) = arg.index {
-                let (i, _) = argmap.insert_full((arg_index, ArgumentType::Usize));
+                let (i, _) = argmap.insert_full((arg_index, ArgumentType::Usize), arg.span);
                 let count_param = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
                     sp,
                     hir::LangItem::FormatCount,
@@ -132,12 +291,14 @@ fn make_format_spec<'hir>(
     ctx: &mut LoweringContext<'_, 'hir>,
     sp: Span,
     placeholder: &FormatPlaceholder,
-    argmap: &mut FxIndexSet<(usize, ArgumentType)>,
+    argmap: &mut FxIndexMap<(usize, ArgumentType), Option<Span>>,
 ) -> hir::Expr<'hir> {
     let position = match placeholder.argument.index {
         Ok(arg_index) => {
-            let (i, _) =
-                argmap.insert_full((arg_index, ArgumentType::Format(placeholder.format_trait)));
+            let (i, _) = argmap.insert_full(
+                (arg_index, ArgumentType::Format(placeholder.format_trait)),
+                placeholder.span,
+            );
             ctx.expr_usize(sp, i)
         }
         Err(_) => ctx.expr(
@@ -189,11 +350,26 @@ fn expand_format_args<'hir>(
     ctx: &mut LoweringContext<'_, 'hir>,
     macsp: Span,
     fmt: &FormatArgs,
+    allow_const: bool,
 ) -> hir::ExprKind<'hir> {
+    let mut incomplete_lit = String::new();
     let lit_pieces =
         ctx.arena.alloc_from_iter(fmt.template.iter().enumerate().filter_map(|(i, piece)| {
             match piece {
-                &FormatArgsPiece::Literal(s) => Some(ctx.expr_str(fmt.span, s)),
+                &FormatArgsPiece::Literal(s) => {
+                    // Coalesce adjacent literal pieces.
+                    if let Some(FormatArgsPiece::Literal(_)) = fmt.template.get(i + 1) {
+                        incomplete_lit.push_str(s.as_str());
+                        None
+                    } else if !incomplete_lit.is_empty() {
+                        incomplete_lit.push_str(s.as_str());
+                        let s = Symbol::intern(&incomplete_lit);
+                        incomplete_lit.clear();
+                        Some(ctx.expr_str(fmt.span, s))
+                    } else {
+                        Some(ctx.expr_str(fmt.span, s))
+                    }
+                }
                 &FormatArgsPiece::Placeholder(_) => {
                     // Inject empty string before placeholders when not already preceded by a literal piece.
                     if i == 0 || matches!(fmt.template[i - 1], FormatArgsPiece::Placeholder(_)) {
@@ -212,7 +388,7 @@ fn expand_format_args<'hir>(
 
     // Create a list of all _unique_ (argument, format trait) combinations.
     // E.g. "{0} {0:x} {0} {1}" -> [(0, Display), (0, LowerHex), (1, Display)]
-    let mut argmap = FxIndexSet::default();
+    let mut argmap = FxIndexMap::default();
     for piece in &fmt.template {
         let FormatArgsPiece::Placeholder(placeholder) = piece else { continue };
         if placeholder.format_options != Default::default() {
@@ -220,7 +396,10 @@ fn expand_format_args<'hir>(
             use_format_options = true;
         }
         if let Ok(index) = placeholder.argument.index {
-            if !argmap.insert((index, ArgumentType::Format(placeholder.format_trait))) {
+            if argmap
+                .insert((index, ArgumentType::Format(placeholder.format_trait)), placeholder.span)
+                .is_some()
+            {
                 // Duplicate (argument, format trait) combination,
                 // which we'll only put once in the args array.
                 use_format_options = true;
@@ -244,6 +423,18 @@ fn expand_format_args<'hir>(
 
     let arguments = fmt.arguments.all_args();
 
+    if allow_const && arguments.is_empty() && argmap.is_empty() {
+        // Generate:
+        //     <core::fmt::Arguments>::new_const(lit_pieces)
+        let new = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
+            macsp,
+            hir::LangItem::FormatArguments,
+            sym::new_const,
+        ));
+        let new_args = ctx.arena.alloc_from_iter([lit_pieces]);
+        return hir::ExprKind::Call(new, new_args);
+    }
+
     // If the args array contains exactly all the original arguments once,
     // in order, we can use a simple array instead of a `match` construction.
     // However, if there's a yield point in any argument except the first one,
@@ -252,7 +443,7 @@ fn expand_format_args<'hir>(
     // This is an optimization, speeding up compilation about 1-2% in some cases.
     // See https://github.com/rust-lang/rust/pull/106770#issuecomment-1380790609
     let use_simple_array = argmap.len() == arguments.len()
-        && argmap.iter().enumerate().all(|(i, &(j, _))| i == j)
+        && argmap.iter().enumerate().all(|(i, (&(j, _), _))| i == j)
         && arguments.iter().skip(1).all(|arg| !may_contain_yield_point(&arg.expr));
 
     let args = if use_simple_array {
@@ -266,14 +457,19 @@ fn expand_format_args<'hir>(
         let elements: Vec<_> = arguments
             .iter()
             .zip(argmap)
-            .map(|(arg, (_, ty))| {
-                let sp = arg.expr.span.with_ctxt(macsp.ctxt());
+            .map(|(arg, ((_, ty), placeholder_span))| {
+                let placeholder_span =
+                    placeholder_span.unwrap_or(arg.expr.span).with_ctxt(macsp.ctxt());
+                let arg_span = match arg.kind {
+                    FormatArgumentKind::Captured(_) => placeholder_span,
+                    _ => arg.expr.span.with_ctxt(macsp.ctxt()),
+                };
                 let arg = ctx.lower_expr(&arg.expr);
                 let ref_arg = ctx.arena.alloc(ctx.expr(
-                    sp,
+                    arg_span,
                     hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, arg),
                 ));
-                make_argument(ctx, sp, ref_arg, ty)
+                make_argument(ctx, placeholder_span, ref_arg, ty)
             })
             .collect();
         ctx.expr_array_ref(macsp, ctx.arena.alloc_from_iter(elements))
@@ -289,27 +485,26 @@ fn expand_format_args<'hir>(
         //     }
         let args_ident = Ident::new(sym::args, macsp);
         let (args_pat, args_hir_id) = ctx.pat_ident(macsp, args_ident);
-        let args = ctx.arena.alloc_from_iter(argmap.iter().map(|&(arg_index, ty)| {
-            if let Some(arg) = arguments.get(arg_index) {
-                let sp = arg.expr.span.with_ctxt(macsp.ctxt());
+        let args = ctx.arena.alloc_from_iter(argmap.iter().map(
+            |(&(arg_index, ty), &placeholder_span)| {
+                let arg = &arguments[arg_index];
+                let placeholder_span =
+                    placeholder_span.unwrap_or(arg.expr.span).with_ctxt(macsp.ctxt());
+                let arg_span = match arg.kind {
+                    FormatArgumentKind::Captured(_) => placeholder_span,
+                    _ => arg.expr.span.with_ctxt(macsp.ctxt()),
+                };
                 let args_ident_expr = ctx.expr_ident(macsp, args_ident, args_hir_id);
                 let arg = ctx.arena.alloc(ctx.expr(
-                    sp,
+                    arg_span,
                     hir::ExprKind::Field(
                         args_ident_expr,
                         Ident::new(sym::integer(arg_index), macsp),
                     ),
                 ));
-                make_argument(ctx, sp, arg, ty)
-            } else {
-                ctx.expr(
-                    macsp,
-                    hir::ExprKind::Err(
-                        ctx.tcx.sess.delay_span_bug(macsp, format!("no arg at {arg_index}")),
-                    ),
-                )
-            }
-        }));
+                make_argument(ctx, placeholder_span, arg, ty)
+            },
+        ));
         let elements: Vec<_> = arguments
             .iter()
             .map(|arg| {
@@ -409,3 +604,22 @@ fn may_contain_yield_point(e: &ast::Expr) -> bool {
     visitor.visit_expr(e);
     visitor.0
 }
+
+fn for_all_argument_indexes(template: &mut [FormatArgsPiece], mut f: impl FnMut(&mut usize)) {
+    for piece in template {
+        let FormatArgsPiece::Placeholder(placeholder) = piece else { continue };
+        if let Ok(index) = &mut placeholder.argument.index {
+            f(index);
+        }
+        if let Some(FormatCount::Argument(FormatArgPosition { index: Ok(index), .. })) =
+            &mut placeholder.format_options.width
+        {
+            f(index);
+        }
+        if let Some(FormatCount::Argument(FormatArgPosition { index: Ok(index), .. })) =
+            &mut placeholder.format_options.precision
+        {
+            f(index);
+        }
+    }
+}
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 41295f2b7..f89e254a2 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -12,8 +12,8 @@ use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID};
 use rustc_hir::PredicateOrigin;
-use rustc_index::vec::{Idx, IndexVec};
-use rustc_middle::ty::{DefIdTree, ResolverAstLowering, TyCtxt};
+use rustc_index::vec::{Idx, IndexSlice, IndexVec};
+use rustc_middle::ty::{ResolverAstLowering, TyCtxt};
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::source_map::DesugaringKind;
 use rustc_span::symbol::{kw, sym, Ident};
@@ -25,7 +25,7 @@ use thin_vec::ThinVec;
 pub(super) struct ItemLowerer<'a, 'hir> {
     pub(super) tcx: TyCtxt<'hir>,
     pub(super) resolver: &'a mut ResolverAstLowering,
-    pub(super) ast_index: &'a IndexVec<LocalDefId, AstOwner<'a>>,
+    pub(super) ast_index: &'a IndexSlice<LocalDefId, AstOwner<'a>>,
     pub(super) owners: &'a mut IndexVec<LocalDefId, hir::MaybeOwner<&'hir hir::OwnerInfo<'hir>>>,
 }
 
@@ -229,12 +229,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
                 self.lower_use_tree(use_tree, &prefix, id, vis_span, ident, attrs)
             }
-            ItemKind::Static(t, m, e) => {
+            ItemKind::Static(box ast::StaticItem { ty: t, mutability: m, expr: e }) => {
                 let (ty, body_id) = self.lower_const_item(t, span, e.as_deref());
                 hir::ItemKind::Static(ty, *m, body_id)
             }
-            ItemKind::Const(_, t, e) => {
-                let (ty, body_id) = self.lower_const_item(t, span, e.as_deref());
+            ItemKind::Const(box ast::ConstItem { ty, expr, .. }) => {
+                let (ty, body_id) = self.lower_const_item(ty, span, expr.as_deref());
                 hir::ItemKind::Const(ty, body_id)
             }
             ItemKind::Fn(box Fn {
@@ -708,10 +708,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let trait_item_def_id = hir_id.expect_owner();
 
         let (generics, kind, has_default) = match &i.kind {
-            AssocItemKind::Const(_, ty, default) => {
+            AssocItemKind::Const(box ConstItem { ty, expr, .. }) => {
                 let ty =
                     self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::ConstTy));
-                let body = default.as_ref().map(|x| self.lower_const_body(i.span, Some(x)));
+                let body = expr.as_ref().map(|x| self.lower_const_body(i.span, Some(x)));
                 (hir::Generics::empty(), hir::TraitItemKind::Const(ty, body), body.is_some())
             }
             AssocItemKind::Fn(box Fn { sig, generics, body: None, .. }) => {
@@ -809,7 +809,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         self.lower_attrs(hir_id, &i.attrs);
 
         let (generics, kind) = match &i.kind {
-            AssocItemKind::Const(_, ty, expr) => {
+            AssocItemKind::Const(box ConstItem { ty, expr, .. }) => {
                 let ty =
                     self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::ConstTy));
                 (
@@ -1146,7 +1146,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
             let async_expr = this.make_async_expr(
                 CaptureBy::Value,
-                fn_id,
                 closure_id,
                 None,
                 body.span,
@@ -1180,7 +1179,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 },
             );
 
-            (this.arena.alloc_from_iter(parameters), this.expr(body.span, async_expr))
+            let hir_id = this.lower_node_id(closure_id);
+            this.maybe_forward_track_caller(body.span, fn_id, hir_id);
+            let expr = hir::Expr { hir_id, kind: async_expr, span: this.lower_span(body.span) };
+
+            (this.arena.alloc_from_iter(parameters), expr)
         })
     }
 
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index b20157f2c..f7ae96b7c 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -60,13 +60,13 @@ use rustc_hir::def::{DefKind, LifetimeRes, Namespace, PartialRes, PerNS, Res};
 use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID};
 use rustc_hir::definitions::DefPathData;
 use rustc_hir::{ConstArg, GenericArg, ItemLocalId, ParamName, TraitCandidate};
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{Idx, IndexSlice, IndexVec};
 use rustc_macros::fluent_messages;
 use rustc_middle::{
     span_bug,
     ty::{ResolverAstLowering, TyCtxt},
 };
-use rustc_session::parse::feature_err;
+use rustc_session::parse::{add_feature_diagnostics, feature_err};
 use rustc_span::hygiene::MacroKind;
 use rustc_span::source_map::DesugaringKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
@@ -92,7 +92,7 @@ mod lifetime_collector;
 mod pat;
 mod path;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 struct LoweringContext<'a, 'hir> {
     tcx: TyCtxt<'hir>,
@@ -414,7 +414,7 @@ fn index_crate<'a>(
 /// This hash will then be part of the crate_hash which is stored in the metadata.
 fn compute_hir_hash(
     tcx: TyCtxt<'_>,
-    owners: &IndexVec<LocalDefId, hir::MaybeOwner<&hir::OwnerInfo<'_>>>,
+    owners: &IndexSlice<LocalDefId, hir::MaybeOwner<&hir::OwnerInfo<'_>>>,
 ) -> Fingerprint {
     let mut hir_body_nodes: Vec<_> = owners
         .iter_enumerated()
@@ -435,7 +435,9 @@ fn compute_hir_hash(
 
 pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
     let sess = tcx.sess;
-    tcx.ensure().output_filenames(());
+    // Queries that borrow `resolver_for_lowering`.
+    tcx.ensure_with_value().output_filenames(());
+    tcx.ensure_with_value().early_lint_checks(());
     let (mut resolver, krate) = tcx.resolver_for_lowering(()).steal();
 
     let ast_index = index_crate(&resolver.node_id_to_def_id, &krate);
@@ -463,8 +465,10 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
         rustc_span::hygiene::clear_syntax_context_map();
     }
 
-    let hir_hash = compute_hir_hash(tcx, &owners);
-    hir::Crate { owners, hir_hash }
+    // Don't hash unless necessary, because it's expensive.
+    let opt_hir_hash =
+        if tcx.sess.needs_crate_hash() { Some(compute_hir_hash(tcx, &owners)) } else { None };
+    hir::Crate { owners, opt_hir_hash }
 }
 
 #[derive(Copy, Clone, PartialEq, Debug)]
@@ -478,7 +482,7 @@ enum ParamMode {
 }
 
 enum ParenthesizedGenericArgs {
-    Ok,
+    ParenSugar,
     Err,
 }
 
@@ -657,42 +661,33 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         bodies.sort_by_key(|(k, _)| *k);
         let bodies = SortedMap::from_presorted_elements(bodies);
-        let (hash_including_bodies, hash_without_bodies) = self.hash_owner(node, &bodies);
-        let (nodes, parenting) =
-            index::index_hir(self.tcx.sess, &*self.tcx.definitions_untracked(), node, &bodies);
-        let nodes = hir::OwnerNodes { hash_including_bodies, hash_without_bodies, nodes, bodies };
-        let attrs = {
-            let hash = self.tcx.with_stable_hashing_context(|mut hcx| {
+
+        // Don't hash unless necessary, because it's expensive.
+        let (opt_hash_including_bodies, attrs_hash) = if self.tcx.sess.needs_crate_hash() {
+            self.tcx.with_stable_hashing_context(|mut hcx| {
+                let mut stable_hasher = StableHasher::new();
+                hcx.with_hir_bodies(node.def_id(), &bodies, |hcx| {
+                    node.hash_stable(hcx, &mut stable_hasher)
+                });
+                let h1 = stable_hasher.finish();
+
                 let mut stable_hasher = StableHasher::new();
                 attrs.hash_stable(&mut hcx, &mut stable_hasher);
-                stable_hasher.finish()
-            });
-            hir::AttributeMap { map: attrs, hash }
+                let h2 = stable_hasher.finish();
+
+                (Some(h1), Some(h2))
+            })
+        } else {
+            (None, None)
         };
+        let (nodes, parenting) =
+            index::index_hir(self.tcx.sess, &*self.tcx.definitions_untracked(), node, &bodies);
+        let nodes = hir::OwnerNodes { opt_hash_including_bodies, nodes, bodies };
+        let attrs = hir::AttributeMap { map: attrs, opt_hash: attrs_hash };
 
         self.arena.alloc(hir::OwnerInfo { nodes, parenting, attrs, trait_map })
     }
 
-    /// Hash the HIR node twice, one deep and one shallow hash. This allows to differentiate
-    /// queries which depend on the full HIR tree and those which only depend on the item signature.
-    fn hash_owner(
-        &mut self,
-        node: hir::OwnerNode<'hir>,
-        bodies: &SortedMap<hir::ItemLocalId, &'hir hir::Body<'hir>>,
-    ) -> (Fingerprint, Fingerprint) {
-        self.tcx.with_stable_hashing_context(|mut hcx| {
-            let mut stable_hasher = StableHasher::new();
-            hcx.with_hir_bodies(node.def_id(), bodies, |hcx| {
-                node.hash_stable(hcx, &mut stable_hasher)
-            });
-            let hash_including_bodies = stable_hasher.finish();
-            let mut stable_hasher = StableHasher::new();
-            hcx.without_hir_bodies(|hcx| node.hash_stable(hcx, &mut stable_hasher));
-            let hash_without_bodies = stable_hasher.finish();
-            (hash_including_bodies, hash_without_bodies)
-        })
-    }
-
     /// This method allocates a new `HirId` for the given `NodeId` and stores it in
     /// the `LoweringContext`'s `NodeId => HirId` map.
     /// Take care not to call this method if the resulting `HirId` is then not
@@ -993,13 +988,60 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     self.lower_angle_bracketed_parameter_data(data, ParamMode::Explicit, itctx).0
                 }
                 GenericArgs::Parenthesized(data) => {
-                    self.emit_bad_parenthesized_trait_in_assoc_ty(data);
-                    self.lower_angle_bracketed_parameter_data(
-                        &data.as_angle_bracketed_args(),
-                        ParamMode::Explicit,
-                        itctx,
-                    )
-                    .0
+                    if data.inputs.is_empty() && matches!(data.output, FnRetTy::Default(..)) {
+                        let parenthesized = if self.tcx.features().return_type_notation {
+                            hir::GenericArgsParentheses::ReturnTypeNotation
+                        } else {
+                            self.emit_bad_parenthesized_trait_in_assoc_ty(data);
+                            hir::GenericArgsParentheses::No
+                        };
+                        GenericArgsCtor {
+                            args: Default::default(),
+                            bindings: &[],
+                            parenthesized,
+                            span: data.inputs_span,
+                        }
+                    } else if let Some(first_char) = constraint.ident.as_str().chars().next()
+                        && first_char.is_ascii_lowercase()
+                    {
+                        let mut err = if !data.inputs.is_empty() {
+                            self.tcx.sess.create_err(errors::BadReturnTypeNotation::Inputs {
+                                span: data.inputs_span,
+                            })
+                        } else if let FnRetTy::Ty(ty) = &data.output {
+                            self.tcx.sess.create_err(errors::BadReturnTypeNotation::Output {
+                                span: data.inputs_span.shrink_to_hi().to(ty.span),
+                            })
+                        } else {
+                            unreachable!("inputs are empty and return type is not provided")
+                        };
+                        if !self.tcx.features().return_type_notation
+                            && self.tcx.sess.is_nightly_build()
+                        {
+                            add_feature_diagnostics(
+                                &mut err,
+                                &self.tcx.sess.parse_sess,
+                                sym::return_type_notation,
+                            );
+                        }
+                        err.emit();
+                        GenericArgsCtor {
+                            args: Default::default(),
+                            bindings: &[],
+                            parenthesized: hir::GenericArgsParentheses::ReturnTypeNotation,
+                            span: data.span,
+                        }
+                    } else {
+                        self.emit_bad_parenthesized_trait_in_assoc_ty(data);
+                        // FIXME(return_type_notation): we could issue a feature error
+                        // if the parens are empty and there's no return type.
+                        self.lower_angle_bracketed_parameter_data(
+                            &data.as_angle_bracketed_args(),
+                            ParamMode::Explicit,
+                            itctx,
+                        )
+                        .0
+                    }
                 }
             };
             gen_args_ctor.into_generic_args(self)
@@ -2080,7 +2122,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let future_args = self.arena.alloc(hir::GenericArgs {
             args: &[],
             bindings: arena_vec![self; self.output_ty_binding(span, output_ty)],
-            parenthesized: false,
+            parenthesized: hir::GenericArgsParentheses::No,
             span_ext: DUMMY_SP,
         });
 
@@ -2190,7 +2232,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             def_id: self.local_def_id(param.id),
             name,
             span: self.lower_span(param.span()),
-            pure_wrt_drop: self.tcx.sess.contains_name(&param.attrs, sym::may_dangle),
+            pure_wrt_drop: attr::contains_name(&param.attrs, sym::may_dangle),
             kind,
             colon_span: param.colon_span.map(|s| self.lower_span(s)),
             source,
@@ -2600,13 +2642,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 struct GenericArgsCtor<'hir> {
     args: SmallVec<[hir::GenericArg<'hir>; 4]>,
     bindings: &'hir [hir::TypeBinding<'hir>],
-    parenthesized: bool,
+    parenthesized: hir::GenericArgsParentheses,
     span: Span,
 }
 
 impl<'hir> GenericArgsCtor<'hir> {
     fn is_empty(&self) -> bool {
-        self.args.is_empty() && self.bindings.is_empty() && !self.parenthesized
+        self.args.is_empty()
+            && self.bindings.is_empty()
+            && self.parenthesized == hir::GenericArgsParentheses::No
     }
 
     fn into_generic_args(self, this: &LoweringContext<'_, 'hir>) -> &'hir hir::GenericArgs<'hir> {
diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs
index 592fc5aa6..8eb84c036 100644
--- a/compiler/rustc_ast_lowering/src/path.rs
+++ b/compiler/rustc_ast_lowering/src/path.rs
@@ -51,7 +51,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     let parenthesized_generic_args = match base_res {
                         // `a::b::Trait(Args)`
                         Res::Def(DefKind::Trait, _) if i + 1 == proj_start => {
-                            ParenthesizedGenericArgs::Ok
+                            ParenthesizedGenericArgs::ParenSugar
                         }
                         // `a::b::Trait(Args)::TraitItem`
                         Res::Def(DefKind::AssocFn, _)
@@ -59,10 +59,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         | Res::Def(DefKind::AssocTy, _)
                             if i + 2 == proj_start =>
                         {
-                            ParenthesizedGenericArgs::Ok
+                            ParenthesizedGenericArgs::ParenSugar
                         }
                         // Avoid duplicated errors.
-                        Res::Err => ParenthesizedGenericArgs::Ok,
+                        Res::Err => ParenthesizedGenericArgs::ParenSugar,
                         // An error
                         _ => ParenthesizedGenericArgs::Err,
                     };
@@ -180,7 +180,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     self.lower_angle_bracketed_parameter_data(data, param_mode, itctx)
                 }
                 GenericArgs::Parenthesized(data) => match parenthesized_generic_args {
-                    ParenthesizedGenericArgs::Ok => {
+                    ParenthesizedGenericArgs::ParenSugar => {
                         self.lower_parenthesized_parameter_data(data, itctx)
                     }
                     ParenthesizedGenericArgs::Err => {
@@ -224,7 +224,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 GenericArgsCtor {
                     args: Default::default(),
                     bindings: &[],
-                    parenthesized: false,
+                    parenthesized: hir::GenericArgsParentheses::No,
                     span: path_span.shrink_to_hi(),
                 },
                 param_mode == ParamMode::Optional,
@@ -233,7 +233,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         let has_lifetimes =
             generic_args.args.iter().any(|arg| matches!(arg, GenericArg::Lifetime(_)));
-        if !generic_args.parenthesized && !has_lifetimes {
+
+        // FIXME(return_type_notation): Is this correct? I think so.
+        if generic_args.parenthesized != hir::GenericArgsParentheses::ParenSugar && !has_lifetimes {
             self.maybe_insert_elided_lifetimes_in_path(
                 path_span,
                 segment.id,
@@ -328,7 +330,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             AngleBracketedArg::Constraint(c) => Some(self.lower_assoc_ty_constraint(c, itctx)),
             AngleBracketedArg::Arg(_) => None,
         }));
-        let ctor = GenericArgsCtor { args, bindings, parenthesized: false, span: data.span };
+        let ctor = GenericArgsCtor {
+            args,
+            bindings,
+            parenthesized: hir::GenericArgsParentheses::No,
+            span: data.span,
+        };
         (ctor, !has_non_lt_args && param_mode == ParamMode::Optional)
     }
 
@@ -376,7 +383,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             GenericArgsCtor {
                 args,
                 bindings: arena_vec![self; binding],
-                parenthesized: true,
+                parenthesized: hir::GenericArgsParentheses::ParenSugar,
                 span: data.inputs_span,
             },
             false,
@@ -396,7 +403,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let gen_args = self.arena.alloc(hir::GenericArgs {
             args,
             bindings,
-            parenthesized: false,
+            parenthesized: hir::GenericArgsParentheses::No,
             span_ext: DUMMY_SP,
         });
         hir::TypeBinding {
diff --git a/compiler/rustc_ast_passes/locales/en-US.ftl b/compiler/rustc_ast_passes/messages.ftl
index 747bd52b2..a349fe6a3 100644
--- a/compiler/rustc_ast_passes/locales/en-US.ftl
+++ b/compiler/rustc_ast_passes/messages.ftl
@@ -17,9 +17,10 @@ ast_passes_keyword_lifetime =
 ast_passes_invalid_label =
     invalid label name `{$name}`
 
-ast_passes_invalid_visibility =
-    unnecessary visibility qualifier
-    .implied = `pub` not permitted here because it's implied
+ast_passes_visibility_not_permitted =
+    visibility qualifiers are not permitted here
+    .enum_variant = enum variants and their fields always share the visibility of the enum they are in
+    .trait_impl = trait items always share the visibility of their trait
     .individual_impl_items = place qualifiers on individual impl items instead
     .individual_foreign_items = place qualifiers on individual foreign items instead
 
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 2cc009410..c79626ccd 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -9,10 +9,10 @@
 use itertools::{Either, Itertools};
 use rustc_ast::ptr::P;
 use rustc_ast::visit::{self, AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor};
-use rustc_ast::walk_list;
 use rustc_ast::*;
+use rustc_ast::{walk_list, StaticItem};
 use rustc_ast_pretty::pprust::{self, State};
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_macros::Subdiagnostic;
 use rustc_parse::validate_attr;
 use rustc_session::lint::builtin::{
@@ -240,16 +240,12 @@ impl<'a> AstValidator<'a> {
         }
     }
 
-    fn invalid_visibility(&self, vis: &Visibility, note: Option<errors::InvalidVisibilityNote>) {
+    fn visibility_not_permitted(&self, vis: &Visibility, note: errors::VisibilityNotPermittedNote) {
         if let VisibilityKind::Inherited = vis.kind {
             return;
         }
 
-        self.session.emit_err(errors::InvalidVisibility {
-            span: vis.span,
-            implied: vis.kind.is_pub().then_some(vis.span),
-            note,
-        });
+        self.session.emit_err(errors::VisibilityNotPermitted { span: vis.span, note });
     }
 
     fn check_decl_no_pat(decl: &FnDecl, mut report_err: impl FnMut(Span, Option<Ident>, bool)) {
@@ -643,7 +639,7 @@ fn validate_generic_param_order(
     span: Span,
 ) {
     let mut max_param: Option<ParamKindOrd> = None;
-    let mut out_of_order = FxHashMap::default();
+    let mut out_of_order = FxIndexMap::default();
     let mut param_idents = Vec::with_capacity(generics.len());
 
     for (idx, param) in generics.iter().enumerate() {
@@ -691,7 +687,7 @@ fn validate_generic_param_order(
                 GenericParamKind::Lifetime => (),
                 GenericParamKind::Const { ty: _, kw_span: _, default: Some(default) } => {
                     ordered_params += " = ";
-                    ordered_params += &pprust::expr_to_string(&*default.value);
+                    ordered_params += &pprust::expr_to_string(&default.value);
                 }
                 GenericParamKind::Const { ty: _, kw_span: _, default: None } => (),
             }
@@ -799,11 +795,11 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
     }
 
     fn visit_item(&mut self, item: &'a Item) {
-        if item.attrs.iter().any(|attr| self.session.is_proc_macro_attr(attr)) {
+        if item.attrs.iter().any(|attr| attr.is_proc_macro_attr()) {
             self.has_proc_macro_decls = true;
         }
 
-        if self.session.contains_name(&item.attrs, sym::no_mangle) {
+        if attr::contains_name(&item.attrs, sym::no_mangle) {
             self.check_nomangle_item_asciionly(item.ident, item.span);
         }
 
@@ -819,7 +815,10 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 items,
             }) => {
                 self.with_in_trait_impl(true, Some(*constness), |this| {
-                    this.invalid_visibility(&item.vis, None);
+                    this.visibility_not_permitted(
+                        &item.vis,
+                        errors::VisibilityNotPermittedNote::TraitImpl,
+                    );
                     if let TyKind::Err = self_ty.kind {
                         this.err_handler().emit_err(errors::ObsoleteAuto { span: item.span });
                     }
@@ -866,9 +865,9 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                         only_trait: only_trait.then_some(()),
                     };
 
-                self.invalid_visibility(
+                self.visibility_not_permitted(
                     &item.vis,
-                    Some(errors::InvalidVisibilityNote::IndividualImplItems),
+                    errors::VisibilityNotPermittedNote::IndividualImplItems,
                 );
                 if let &Unsafe::Yes(span) = unsafety {
                     self.err_handler().emit_err(errors::InherentImplCannotUnsafe {
@@ -924,9 +923,9 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             }
             ItemKind::ForeignMod(ForeignMod { abi, unsafety, .. }) => {
                 let old_item = mem::replace(&mut self.extern_mod, Some(item));
-                self.invalid_visibility(
+                self.visibility_not_permitted(
                     &item.vis,
-                    Some(errors::InvalidVisibilityNote::IndividualForeignItems),
+                    errors::VisibilityNotPermittedNote::IndividualForeignItems,
                 );
                 if let &Unsafe::Yes(span) = unsafety {
                     self.err_handler().emit_err(errors::UnsafeItem { span, kind: "extern block" });
@@ -940,9 +939,15 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             }
             ItemKind::Enum(def, _) => {
                 for variant in &def.variants {
-                    self.invalid_visibility(&variant.vis, None);
+                    self.visibility_not_permitted(
+                        &variant.vis,
+                        errors::VisibilityNotPermittedNote::EnumVariant,
+                    );
                     for field in variant.data.fields() {
-                        self.invalid_visibility(&field.vis, None);
+                        self.visibility_not_permitted(
+                            &field.vis,
+                            errors::VisibilityNotPermittedNote::EnumVariant,
+                        );
                     }
                 }
             }
@@ -973,7 +978,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 }
                 // Ensure that `path` attributes on modules are recorded as used (cf. issue #35584).
                 if !matches!(mod_kind, ModKind::Loaded(_, Inline::Yes, _))
-                    && !self.session.contains_name(&item.attrs, sym::path)
+                    && !attr::contains_name(&item.attrs, sym::path)
                 {
                     self.check_mod_file_item_asciionly(item.ident);
                 }
@@ -983,14 +988,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     self.err_handler().emit_err(errors::FieldlessUnion { span: item.span });
                 }
             }
-            ItemKind::Const(def, .., None) => {
-                self.check_defaultness(item.span, *def);
+            ItemKind::Const(box ConstItem { defaultness, expr: None, .. }) => {
+                self.check_defaultness(item.span, *defaultness);
                 self.session.emit_err(errors::ConstWithoutBody {
                     span: item.span,
                     replace_span: self.ending_semi_or_hi(item.span),
                 });
             }
-            ItemKind::Static(.., None) => {
+            ItemKind::Static(box StaticItem { expr: None, .. }) => {
                 self.session.emit_err(errors::StaticWithoutBody {
                     span: item.span,
                     replace_span: self.ending_semi_or_hi(item.span),
@@ -1248,7 +1253,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
     }
 
     fn visit_assoc_item(&mut self, item: &'a AssocItem, ctxt: AssocCtxt) {
-        if self.session.contains_name(&item.attrs, sym::no_mangle) {
+        if attr::contains_name(&item.attrs, sym::no_mangle) {
             self.check_nomangle_item_asciionly(item.ident, item.span);
         }
 
@@ -1258,13 +1263,11 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
         if ctxt == AssocCtxt::Impl {
             match &item.kind {
-                AssocItemKind::Const(_, _, body) => {
-                    if body.is_none() {
-                        self.session.emit_err(errors::AssocConstWithoutBody {
-                            span: item.span,
-                            replace_span: self.ending_semi_or_hi(item.span),
-                        });
-                    }
+                AssocItemKind::Const(box ConstItem { expr: None, .. }) => {
+                    self.session.emit_err(errors::AssocConstWithoutBody {
+                        span: item.span,
+                        replace_span: self.ending_semi_or_hi(item.span),
+                    });
                 }
                 AssocItemKind::Fn(box Fn { body, .. }) => {
                     if body.is_none() {
@@ -1302,7 +1305,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         }
 
         if ctxt == AssocCtxt::Trait || self.in_trait_impl {
-            self.invalid_visibility(&item.vis, None);
+            self.visibility_not_permitted(&item.vis, errors::VisibilityNotPermittedNote::TraitImpl);
             if let AssocItemKind::Fn(box Fn { sig, .. }) = &item.kind {
                 self.check_trait_fn_not_const(sig.header.constness);
             }
@@ -1392,11 +1395,13 @@ fn deny_equality_constraints(
                                             }
                                         },
                                         empty_args => {
-                                            *empty_args = AngleBracketedArgs {
-                                                span: ident.span,
-                                                args: thin_vec![arg],
-                                            }
-                                            .into();
+                                            *empty_args = Some(
+                                                AngleBracketedArgs {
+                                                    span: ident.span,
+                                                    args: thin_vec![arg],
+                                                }
+                                                .into(),
+                                            );
                                         }
                                     }
                                     err.assoc = Some(errors::AssociatedSuggestion {
diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs
index d007097d9..27bbd2379 100644
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@@ -42,18 +42,20 @@ pub struct InvalidLabel {
 }
 
 #[derive(Diagnostic)]
-#[diag(ast_passes_invalid_visibility, code = "E0449")]
-pub struct InvalidVisibility {
+#[diag(ast_passes_visibility_not_permitted, code = "E0449")]
+pub struct VisibilityNotPermitted {
     #[primary_span]
     pub span: Span,
-    #[label(ast_passes_implied)]
-    pub implied: Option<Span>,
     #[subdiagnostic]
-    pub note: Option<InvalidVisibilityNote>,
+    pub note: VisibilityNotPermittedNote,
 }
 
 #[derive(Subdiagnostic)]
-pub enum InvalidVisibilityNote {
+pub enum VisibilityNotPermittedNote {
+    #[note(ast_passes_enum_variant)]
+    EnumVariant,
+    #[note(ast_passes_trait_impl)]
+    TraitImpl,
     #[note(ast_passes_individual_impl_items)]
     IndividualImplItems,
     #[note(ast_passes_individual_foreign_items)]
diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs
index 96042ea30..17bcd24ee 100644
--- a/compiler/rustc_ast_passes/src/feature_gate.rs
+++ b/compiler/rustc_ast_passes/src/feature_gate.rs
@@ -1,6 +1,6 @@
 use rustc_ast as ast;
 use rustc_ast::visit::{self, AssocCtxt, FnCtxt, FnKind, Visitor};
-use rustc_ast::{AssocConstraint, AssocConstraintKind, NodeId};
+use rustc_ast::{attr, AssocConstraint, AssocConstraintKind, NodeId};
 use rustc_ast::{PatKind, RangeEnd};
 use rustc_errors::{Applicability, StashKey};
 use rustc_feature::{AttributeGate, BuiltinAttribute, Features, GateIssue, BUILTIN_ATTRIBUTE_MAP};
@@ -121,24 +121,34 @@ impl<'a> PostExpansionVisitor<'a> {
     }
 
     /// Feature gate `impl Trait` inside `type Alias = $type_expr;`.
-    fn check_impl_trait(&self, ty: &ast::Ty) {
+    fn check_impl_trait(&self, ty: &ast::Ty, in_associated_ty: bool) {
         struct ImplTraitVisitor<'a> {
             vis: &'a PostExpansionVisitor<'a>,
+            in_associated_ty: bool,
         }
         impl Visitor<'_> for ImplTraitVisitor<'_> {
             fn visit_ty(&mut self, ty: &ast::Ty) {
                 if let ast::TyKind::ImplTrait(..) = ty.kind {
-                    gate_feature_post!(
-                        &self.vis,
-                        type_alias_impl_trait,
-                        ty.span,
-                        "`impl Trait` in type aliases is unstable"
-                    );
+                    if self.in_associated_ty {
+                        gate_feature_post!(
+                            &self.vis,
+                            impl_trait_in_assoc_type,
+                            ty.span,
+                            "`impl Trait` in associated types is unstable"
+                        );
+                    } else {
+                        gate_feature_post!(
+                            &self.vis,
+                            type_alias_impl_trait,
+                            ty.span,
+                            "`impl Trait` in type aliases is unstable"
+                        );
+                    }
                 }
                 visit::walk_ty(self, ty);
             }
         }
-        ImplTraitVisitor { vis: self }.visit_ty(ty);
+        ImplTraitVisitor { vis: self, in_associated_ty }.visit_ty(ty);
     }
 
     fn check_late_bound_lifetime_defs(&self, params: &[ast::GenericParam]) {
@@ -232,7 +242,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             }
 
             ast::ItemKind::Fn(..) => {
-                if self.sess.contains_name(&i.attrs, sym::start) {
+                if attr::contains_name(&i.attrs, sym::start) {
                     gate_feature_post!(
                         &self,
                         start,
@@ -245,7 +255,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             }
 
             ast::ItemKind::Struct(..) => {
-                for attr in self.sess.filter_by_name(&i.attrs, sym::repr) {
+                for attr in attr::filter_by_name(&i.attrs, sym::repr) {
                     for item in attr.meta_item_list().unwrap_or_else(ThinVec::new) {
                         if item.has_name(sym::simd) {
                             gate_feature_post!(
@@ -294,7 +304,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             }
 
             ast::ItemKind::TyAlias(box ast::TyAlias { ty: Some(ty), .. }) => {
-                self.check_impl_trait(&ty)
+                self.check_impl_trait(&ty, false)
             }
 
             _ => {}
@@ -306,7 +316,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
     fn visit_foreign_item(&mut self, i: &'a ast::ForeignItem) {
         match i.kind {
             ast::ForeignItemKind::Fn(..) | ast::ForeignItemKind::Static(..) => {
-                let link_name = self.sess.first_attr_value_str_by_name(&i.attrs, sym::link_name);
+                let link_name = attr::first_attr_value_str_by_name(&i.attrs, sym::link_name);
                 let links_to_llvm =
                     link_name.map_or(false, |val| val.as_str().starts_with("llvm."));
                 if links_to_llvm {
@@ -337,9 +347,6 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             ast::TyKind::Never => {
                 gate_feature_post!(&self, never_type, ty.span, "the `!` type is experimental");
             }
-            ast::TyKind::TraitObject(_, ast::TraitObjectSyntax::DynStar, ..) => {
-                gate_feature_post!(&self, dyn_star, ty.span, "dyn* trait objects are unstable");
-            }
             _ => {}
         }
         visit::walk_ty(self, ty)
@@ -395,14 +402,6 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 
     fn visit_expr(&mut self, e: &'a ast::Expr) {
         match e.kind {
-            ast::ExprKind::Box(_) => {
-                gate_feature_post!(
-                    &self,
-                    box_syntax,
-                    e.span,
-                    "box expression syntax is experimental; you can call `Box::new` instead"
-                );
-            }
             ast::ExprKind::Type(..) => {
                 if self.sess.parse_sess.span_diagnostic.err_count() == 0 {
                     // To avoid noise about type ascription in common syntax errors,
@@ -415,24 +414,19 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                     );
                 } else {
                     // And if it isn't, cancel the early-pass warning.
-                    self.sess
+                    if let Some(err) = self
+                        .sess
                         .parse_sess
                         .span_diagnostic
                         .steal_diagnostic(e.span, StashKey::EarlySyntaxWarning)
-                        .map(|err| err.cancel());
+                    {
+                        err.cancel()
+                    }
                 }
             }
             ast::ExprKind::TryBlock(_) => {
                 gate_feature_post!(&self, try_blocks, e.span, "`try` expression is experimental");
             }
-            ast::ExprKind::Closure(box ast::Closure { constness: ast::Const::Yes(_), .. }) => {
-                gate_feature_post!(
-                    &self,
-                    const_closures,
-                    e.span,
-                    "const closures are experimental"
-                );
-            }
             _ => {}
         }
         visit::walk_expr(self, e)
@@ -501,12 +495,24 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 
     fn visit_assoc_constraint(&mut self, constraint: &'a AssocConstraint) {
         if let AssocConstraintKind::Bound { .. } = constraint.kind {
-            gate_feature_post!(
-                &self,
-                associated_type_bounds,
-                constraint.span,
-                "associated type bounds are unstable"
-            )
+            if let Some(ast::GenericArgs::Parenthesized(args)) = constraint.gen_args.as_ref()
+                && args.inputs.is_empty()
+                && matches!(args.output, ast::FnRetTy::Default(..))
+            {
+                gate_feature_post!(
+                    &self,
+                    return_type_notation,
+                    constraint.span,
+                    "return type notation is experimental"
+                );
+            } else {
+                gate_feature_post!(
+                    &self,
+                    associated_type_bounds,
+                    constraint.span,
+                    "associated type bounds are unstable"
+                );
+            }
         }
         visit::walk_assoc_constraint(self, constraint)
     }
@@ -524,7 +530,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                     );
                 }
                 if let Some(ty) = ty {
-                    self.check_impl_trait(ty);
+                    self.check_impl_trait(ty, true);
                 }
                 false
             }
@@ -594,6 +600,8 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
     gate_all!(inline_const_pat, "inline-const in pattern position is experimental");
     gate_all!(associated_const_equality, "associated const equality is incomplete");
     gate_all!(yeet_expr, "`do yeet` expression is experimental");
+    gate_all!(dyn_star, "`dyn*` trait objects are experimental");
+    gate_all!(const_closures, "const closures are experimental");
 
     // All uses of `gate_all!` below this point were added in #65742,
     // and subsequently disabled (with the non-early gating readded).
@@ -609,11 +617,11 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
 
     gate_all!(trait_alias, "trait aliases are experimental");
     gate_all!(associated_type_bounds, "associated type bounds are unstable");
+    gate_all!(return_type_notation, "return type notation is experimental");
     gate_all!(decl_macro, "`macro` is experimental");
     gate_all!(box_patterns, "box pattern syntax is experimental");
     gate_all!(exclusive_range_pattern, "exclusive range pattern syntax is experimental");
     gate_all!(try_blocks, "`try` blocks are unstable");
-    gate_all!(box_syntax, "box expression syntax is experimental; you can call `Box::new` instead");
     gate_all!(type_ascription, "type ascription is experimental");
 
     visit::walk_crate(&mut visitor, krate);
diff --git a/compiler/rustc_ast_passes/src/lib.rs b/compiler/rustc_ast_passes/src/lib.rs
index b9dcaee23..e2c666604 100644
--- a/compiler/rustc_ast_passes/src/lib.rs
+++ b/compiler/rustc_ast_passes/src/lib.rs
@@ -4,7 +4,6 @@
 //!
 //! The crate also contains other misc AST visitors, e.g. `node_count` and `show_span`.
 
-#![allow(rustc::potential_query_instability)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]
 #![feature(iter_is_partitioned)]
@@ -22,4 +21,4 @@ pub mod feature_gate;
 pub mod node_count;
 pub mod show_span;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
index 694d688bf..849336c86 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -686,7 +686,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
     fn bclose_maybe_open(&mut self, span: rustc_span::Span, empty: bool, close_box: bool) {
         let has_comment = self.maybe_print_comment(span.hi());
         if !empty || has_comment {
-            self.break_offset_if_not_bol(1, -(INDENT_UNIT as isize));
+            self.break_offset_if_not_bol(1, -INDENT_UNIT);
         }
         self.word("}");
         if close_box {
@@ -984,7 +984,9 @@ impl<'a> State<'a> {
 
     pub fn print_assoc_constraint(&mut self, constraint: &ast::AssocConstraint) {
         self.print_ident(constraint.ident);
-        constraint.gen_args.as_ref().map(|args| self.print_generic_args(args, false));
+        if let Some(args) = constraint.gen_args.as_ref() {
+            self.print_generic_args(args, false)
+        }
         self.space();
         match &constraint.kind {
             ast::AssocConstraintKind::Equality { term } => {
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
index cacfe9eb2..776bf5424 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
@@ -244,6 +244,10 @@ impl<'a> State<'a> {
             (&ast::ExprKind::Let { .. }, _) if !parser::needs_par_as_let_scrutinee(prec) => {
                 parser::PREC_FORCE_PAREN
             }
+            // For a binary expression like `(match () { _ => a }) OP b`, the parens are required
+            // otherwise the parser would interpret `match () { _ => a }` as a statement,
+            // with the remaining `OP b` not making sense. So we force parens.
+            (&ast::ExprKind::Match(..), _) => parser::PREC_FORCE_PAREN,
             _ => left_prec,
         };
 
@@ -292,10 +296,6 @@ impl<'a> State<'a> {
         self.ibox(INDENT_UNIT);
         self.ann.pre(self, AnnNode::Expr(expr));
         match &expr.kind {
-            ast::ExprKind::Box(expr) => {
-                self.word_space("box");
-                self.print_expr_maybe_paren(expr, parser::PREC_PREFIX);
-            }
             ast::ExprKind::Array(exprs) => {
                 self.print_expr_vec(exprs);
             }
@@ -439,7 +439,7 @@ impl<'a> State<'a> {
                 self.ibox(0);
                 self.print_block_with_attrs(blk, attrs);
             }
-            ast::ExprKind::Async(capture_clause, _, blk) => {
+            ast::ExprKind::Async(capture_clause, blk) => {
                 self.word_nbsp("async");
                 self.print_capture_clause(*capture_clause);
                 // cbox/ibox in analogy to the `ExprKind::Block` arm above
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/item.rs b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
index bf2c73a66..c465f8c94 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/item.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
@@ -2,6 +2,7 @@ use crate::pp::Breaks::Inconsistent;
 use crate::pprust::state::delimited::IterDelimited;
 use crate::pprust::state::{AnnNode, PrintState, State, INDENT_UNIT};
 
+use ast::StaticItem;
 use rustc_ast as ast;
 use rustc_ast::GenericBound;
 use rustc_ast::ModKind;
@@ -156,7 +157,7 @@ impl<'a> State<'a> {
                 self.print_use_tree(tree);
                 self.word(";");
             }
-            ast::ItemKind::Static(ty, mutbl, body) => {
+            ast::ItemKind::Static(box StaticItem { ty, mutability: mutbl, expr: body }) => {
                 let def = ast::Defaultness::Final;
                 self.print_item_const(
                     item.ident,
@@ -167,8 +168,15 @@ impl<'a> State<'a> {
                     def,
                 );
             }
-            ast::ItemKind::Const(def, ty, body) => {
-                self.print_item_const(item.ident, None, ty, body.as_deref(), &item.vis, *def);
+            ast::ItemKind::Const(box ast::ConstItem { defaultness, ty, expr }) => {
+                self.print_item_const(
+                    item.ident,
+                    None,
+                    ty,
+                    expr.as_deref(),
+                    &item.vis,
+                    *defaultness,
+                );
             }
             ast::ItemKind::Fn(box ast::Fn { defaultness, sig, generics, body }) => {
                 self.print_fn_full(
@@ -507,8 +515,8 @@ impl<'a> State<'a> {
             ast::AssocItemKind::Fn(box ast::Fn { defaultness, sig, generics, body }) => {
                 self.print_fn_full(sig, ident, generics, vis, *defaultness, body.as_deref(), attrs);
             }
-            ast::AssocItemKind::Const(def, ty, body) => {
-                self.print_item_const(ident, None, ty, body.as_deref(), vis, *def);
+            ast::AssocItemKind::Const(box ast::ConstItem { defaultness, ty, expr }) => {
+                self.print_item_const(ident, None, ty, expr.as_deref(), vis, *defaultness);
             }
             ast::AssocItemKind::Type(box ast::TyAlias {
                 defaultness,
diff --git a/compiler/rustc_attr/locales/en-US.ftl b/compiler/rustc_attr/messages.ftl
index a7f8c993d..a7f8c993d 100644
--- a/compiler/rustc_attr/locales/en-US.ftl
+++ b/compiler/rustc_attr/messages.ftl
diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index 3d240108b..cb217be66 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -1,6 +1,6 @@
 //! Parsing and validation of builtin attributes
 
-use rustc_ast as ast;
+use rustc_ast::{self as ast, attr};
 use rustc_ast::{Attribute, LitKind, MetaItem, MetaItemKind, MetaItemLit, NestedMetaItem, NodeId};
 use rustc_ast_pretty::pprust;
 use rustc_feature::{find_gated_cfg, is_builtin_attr_name, Features, GatedCfg};
@@ -226,307 +226,95 @@ impl UnstableReason {
     }
 }
 
-/// Collects stability info from all stability attributes in `attrs`.
-/// Returns `None` if no stability attributes are found.
+/// Collects stability info from `stable`/`unstable`/`rustc_allowed_through_unstable_modules`
+/// attributes in `attrs`. Returns `None` if no stability attributes are found.
 pub fn find_stability(
     sess: &Session,
     attrs: &[Attribute],
     item_sp: Span,
-) -> (Option<(Stability, Span)>, Option<(ConstStability, Span)>, Option<(DefaultBodyStability, Span)>)
-{
-    find_stability_generic(sess, attrs.iter(), item_sp)
-}
-
-fn find_stability_generic<'a, I>(
-    sess: &Session,
-    attrs_iter: I,
-    item_sp: Span,
-) -> (Option<(Stability, Span)>, Option<(ConstStability, Span)>, Option<(DefaultBodyStability, Span)>)
-where
-    I: Iterator<Item = &'a Attribute>,
-{
-    use StabilityLevel::*;
-
+) -> Option<(Stability, Span)> {
     let mut stab: Option<(Stability, Span)> = None;
-    let mut const_stab: Option<(ConstStability, Span)> = None;
-    let mut body_stab: Option<(DefaultBodyStability, Span)> = None;
-    let mut promotable = false;
     let mut allowed_through_unstable_modules = false;
 
-    'outer: for attr in attrs_iter {
-        if ![
-            sym::rustc_const_unstable,
-            sym::rustc_const_stable,
-            sym::unstable,
-            sym::stable,
-            sym::rustc_promotable,
-            sym::rustc_allowed_through_unstable_modules,
-            sym::rustc_default_body_unstable,
-        ]
-        .iter()
-        .any(|&s| attr.has_name(s))
-        {
-            continue; // not a stability level
-        }
-
-        let meta = attr.meta();
+    for attr in attrs {
+        match attr.name_or_empty() {
+            sym::rustc_allowed_through_unstable_modules => allowed_through_unstable_modules = true,
+            sym::unstable => {
+                if stab.is_some() {
+                    handle_errors(&sess.parse_sess, attr.span, AttrError::MultipleStabilityLevels);
+                    break;
+                }
 
-        if attr.has_name(sym::rustc_promotable) {
-            promotable = true;
-        } else if attr.has_name(sym::rustc_allowed_through_unstable_modules) {
-            allowed_through_unstable_modules = true;
-        }
-        // attributes with data
-        else if let Some(meta @ MetaItem { kind: MetaItemKind::List(metas), .. }) = &meta {
-            let get = |meta: &MetaItem, item: &mut Option<Symbol>| {
-                if item.is_some() {
-                    handle_errors(
-                        &sess.parse_sess,
-                        meta.span,
-                        AttrError::MultipleItem(pprust::path_to_string(&meta.path)),
-                    );
-                    return false;
+                if let Some((feature, level)) = parse_unstability(sess, attr) {
+                    stab = Some((Stability { level, feature }, attr.span));
                 }
-                if let Some(v) = meta.value_str() {
-                    *item = Some(v);
-                    true
-                } else {
-                    sess.emit_err(session_diagnostics::IncorrectMetaItem { span: meta.span });
-                    false
+            }
+            sym::stable => {
+                if stab.is_some() {
+                    handle_errors(&sess.parse_sess, attr.span, AttrError::MultipleStabilityLevels);
+                    break;
                 }
-            };
+                if let Some((feature, level)) = parse_stability(sess, attr) {
+                    stab = Some((Stability { level, feature }, attr.span));
+                }
+            }
+            _ => {}
+        }
+    }
 
-            let meta_name = meta.name_or_empty();
-            match meta_name {
-                sym::rustc_const_unstable | sym::rustc_default_body_unstable | sym::unstable => {
-                    if meta_name == sym::unstable && stab.is_some() {
-                        handle_errors(
-                            &sess.parse_sess,
-                            attr.span,
-                            AttrError::MultipleStabilityLevels,
-                        );
-                        break;
-                    } else if meta_name == sym::rustc_const_unstable && const_stab.is_some() {
-                        handle_errors(
-                            &sess.parse_sess,
-                            attr.span,
-                            AttrError::MultipleStabilityLevels,
-                        );
-                        break;
-                    } else if meta_name == sym::rustc_default_body_unstable && body_stab.is_some() {
-                        handle_errors(
-                            &sess.parse_sess,
-                            attr.span,
-                            AttrError::MultipleStabilityLevels,
-                        );
-                        break;
-                    }
+    if allowed_through_unstable_modules {
+        match &mut stab {
+            Some((
+                Stability {
+                    level: StabilityLevel::Stable { allowed_through_unstable_modules, .. },
+                    ..
+                },
+                _,
+            )) => *allowed_through_unstable_modules = true,
+            _ => {
+                sess.emit_err(session_diagnostics::RustcAllowedUnstablePairing { span: item_sp });
+            }
+        }
+    }
 
-                    let mut feature = None;
-                    let mut reason = None;
-                    let mut issue = None;
-                    let mut issue_num = None;
-                    let mut is_soft = false;
-                    let mut implied_by = None;
-                    for meta in metas {
-                        let Some(mi) = meta.meta_item() else {
-                            handle_errors(
-                                &sess.parse_sess,
-                                meta.span(),
-                                AttrError::UnsupportedLiteral(UnsupportedLiteralReason::Generic, false),
-                            );
-                            continue 'outer;
-                        };
-                        match mi.name_or_empty() {
-                            sym::feature => {
-                                if !get(mi, &mut feature) {
-                                    continue 'outer;
-                                }
-                            }
-                            sym::reason => {
-                                if !get(mi, &mut reason) {
-                                    continue 'outer;
-                                }
-                            }
-                            sym::issue => {
-                                if !get(mi, &mut issue) {
-                                    continue 'outer;
-                                }
+    stab
+}
 
-                                // These unwraps are safe because `get` ensures the meta item
-                                // is a name/value pair string literal.
-                                issue_num = match issue.unwrap().as_str() {
-                                    "none" => None,
-                                    issue => match issue.parse::<NonZeroU32>() {
-                                        Ok(num) => Some(num),
-                                        Err(err) => {
-                                            sess.emit_err(
-                                                session_diagnostics::InvalidIssueString {
-                                                    span: mi.span,
-                                                    cause: session_diagnostics::InvalidIssueStringCause::from_int_error_kind(
-                                                        mi.name_value_literal_span().unwrap(),
-                                                        err.kind(),
-                                                    ),
-                                                },
-                                            );
-                                            continue 'outer;
-                                        }
-                                    },
-                                };
-                            }
-                            sym::soft => {
-                                if !mi.is_word() {
-                                    sess.emit_err(session_diagnostics::SoftNoArgs {
-                                        span: mi.span,
-                                    });
-                                }
-                                is_soft = true;
-                            }
-                            sym::implied_by => {
-                                if !get(mi, &mut implied_by) {
-                                    continue 'outer;
-                                }
-                            }
-                            _ => {
-                                handle_errors(
-                                    &sess.parse_sess,
-                                    meta.span(),
-                                    AttrError::UnknownMetaItem(
-                                        pprust::path_to_string(&mi.path),
-                                        &["feature", "reason", "issue", "soft"],
-                                    ),
-                                );
-                                continue 'outer;
-                            }
-                        }
-                    }
+/// Collects stability info from `rustc_const_stable`/`rustc_const_unstable`/`rustc_promotable`
+/// attributes in `attrs`. Returns `None` if no stability attributes are found.
+pub fn find_const_stability(
+    sess: &Session,
+    attrs: &[Attribute],
+    item_sp: Span,
+) -> Option<(ConstStability, Span)> {
+    let mut const_stab: Option<(ConstStability, Span)> = None;
+    let mut promotable = false;
 
-                    match (feature, reason, issue) {
-                        (Some(feature), reason, Some(_)) => {
-                            if !rustc_lexer::is_ident(feature.as_str()) {
-                                handle_errors(
-                                    &sess.parse_sess,
-                                    attr.span,
-                                    AttrError::NonIdentFeature,
-                                );
-                                continue;
-                            }
-                            let level = Unstable {
-                                reason: UnstableReason::from_opt_reason(reason),
-                                issue: issue_num,
-                                is_soft,
-                                implied_by,
-                            };
-                            if sym::unstable == meta_name {
-                                stab = Some((Stability { level, feature }, attr.span));
-                            } else if sym::rustc_const_unstable == meta_name {
-                                const_stab = Some((
-                                    ConstStability { level, feature, promotable: false },
-                                    attr.span,
-                                ));
-                            } else if sym::rustc_default_body_unstable == meta_name {
-                                body_stab =
-                                    Some((DefaultBodyStability { level, feature }, attr.span));
-                            } else {
-                                unreachable!("Unknown stability attribute {meta_name}");
-                            }
-                        }
-                        (None, _, _) => {
-                            handle_errors(&sess.parse_sess, attr.span, AttrError::MissingFeature);
-                            continue;
-                        }
-                        _ => {
-                            sess.emit_err(session_diagnostics::MissingIssue { span: attr.span });
-                            continue;
-                        }
-                    }
+    for attr in attrs {
+        match attr.name_or_empty() {
+            sym::rustc_promotable => promotable = true,
+            sym::rustc_const_unstable => {
+                if const_stab.is_some() {
+                    handle_errors(&sess.parse_sess, attr.span, AttrError::MultipleStabilityLevels);
+                    break;
                 }
-                sym::rustc_const_stable | sym::stable => {
-                    if meta_name == sym::stable && stab.is_some() {
-                        handle_errors(
-                            &sess.parse_sess,
-                            attr.span,
-                            AttrError::MultipleStabilityLevels,
-                        );
-                        break;
-                    } else if meta_name == sym::rustc_const_stable && const_stab.is_some() {
-                        handle_errors(
-                            &sess.parse_sess,
-                            attr.span,
-                            AttrError::MultipleStabilityLevels,
-                        );
-                        break;
-                    }
-
-                    let mut feature = None;
-                    let mut since = None;
-                    for meta in metas {
-                        match meta {
-                            NestedMetaItem::MetaItem(mi) => match mi.name_or_empty() {
-                                sym::feature => {
-                                    if !get(mi, &mut feature) {
-                                        continue 'outer;
-                                    }
-                                }
-                                sym::since => {
-                                    if !get(mi, &mut since) {
-                                        continue 'outer;
-                                    }
-                                }
-                                _ => {
-                                    handle_errors(
-                                        &sess.parse_sess,
-                                        meta.span(),
-                                        AttrError::UnknownMetaItem(
-                                            pprust::path_to_string(&mi.path),
-                                            &["feature", "since"],
-                                        ),
-                                    );
-                                    continue 'outer;
-                                }
-                            },
-                            NestedMetaItem::Lit(lit) => {
-                                handle_errors(
-                                    &sess.parse_sess,
-                                    lit.span,
-                                    AttrError::UnsupportedLiteral(
-                                        UnsupportedLiteralReason::Generic,
-                                        false,
-                                    ),
-                                );
-                                continue 'outer;
-                            }
-                        }
-                    }
-
-                    if let Some(s) = since && s.as_str() == VERSION_PLACEHOLDER {
-                        since = Some(rust_version_symbol());
-                    }
 
-                    match (feature, since) {
-                        (Some(feature), Some(since)) => {
-                            let level = Stable { since, allowed_through_unstable_modules: false };
-                            if sym::stable == meta_name {
-                                stab = Some((Stability { level, feature }, attr.span));
-                            } else {
-                                const_stab = Some((
-                                    ConstStability { level, feature, promotable: false },
-                                    attr.span,
-                                ));
-                            }
-                        }
-                        (None, _) => {
-                            handle_errors(&sess.parse_sess, attr.span, AttrError::MissingFeature);
-                            continue;
-                        }
-                        _ => {
-                            handle_errors(&sess.parse_sess, attr.span, AttrError::MissingSince);
-                            continue;
-                        }
-                    }
+                if let Some((feature, level)) = parse_unstability(sess, attr) {
+                    const_stab =
+                        Some((ConstStability { level, feature, promotable: false }, attr.span));
+                }
+            }
+            sym::rustc_const_stable => {
+                if const_stab.is_some() {
+                    handle_errors(&sess.parse_sess, attr.span, AttrError::MultipleStabilityLevels);
+                    break;
+                }
+                if let Some((feature, level)) = parse_stability(sess, attr) {
+                    const_stab =
+                        Some((ConstStability { level, feature, promotable: false }, attr.span));
                 }
-                _ => unreachable!(),
             }
+            _ => {}
         }
     }
 
@@ -538,26 +326,241 @@ where
         }
     }
 
-    if allowed_through_unstable_modules {
-        match &mut stab {
-            Some((
-                Stability {
-                    level: StabilityLevel::Stable { allowed_through_unstable_modules, .. },
-                    ..
-                },
-                _,
-            )) => *allowed_through_unstable_modules = true,
+    const_stab
+}
+
+/// Collects stability info from `rustc_default_body_unstable` attributes in `attrs`.
+/// Returns `None` if no stability attributes are found.
+pub fn find_body_stability(
+    sess: &Session,
+    attrs: &[Attribute],
+) -> Option<(DefaultBodyStability, Span)> {
+    let mut body_stab: Option<(DefaultBodyStability, Span)> = None;
+
+    for attr in attrs {
+        if attr.has_name(sym::rustc_default_body_unstable) {
+            if body_stab.is_some() {
+                handle_errors(&sess.parse_sess, attr.span, AttrError::MultipleStabilityLevels);
+                break;
+            }
+
+            if let Some((feature, level)) = parse_unstability(sess, attr) {
+                body_stab = Some((DefaultBodyStability { level, feature }, attr.span));
+            }
+        }
+    }
+
+    body_stab
+}
+
+/// Read the content of a `stable`/`rustc_const_stable` attribute, and return the feature name and
+/// its stability information.
+fn parse_stability(sess: &Session, attr: &Attribute) -> Option<(Symbol, StabilityLevel)> {
+    let meta = attr.meta()?;
+    let MetaItem { kind: MetaItemKind::List(ref metas), .. } = meta else { return None };
+    let insert_or_error = |meta: &MetaItem, item: &mut Option<Symbol>| {
+        if item.is_some() {
+            handle_errors(
+                &sess.parse_sess,
+                meta.span,
+                AttrError::MultipleItem(pprust::path_to_string(&meta.path)),
+            );
+            return false;
+        }
+        if let Some(v) = meta.value_str() {
+            *item = Some(v);
+            true
+        } else {
+            sess.emit_err(session_diagnostics::IncorrectMetaItem { span: meta.span });
+            false
+        }
+    };
+
+    let mut feature = None;
+    let mut since = None;
+    for meta in metas {
+        let Some(mi) = meta.meta_item() else {
+            handle_errors(
+                &sess.parse_sess,
+                meta.span(),
+                AttrError::UnsupportedLiteral(UnsupportedLiteralReason::Generic, false),
+            );
+            return None;
+        };
+
+        match mi.name_or_empty() {
+            sym::feature => {
+                if !insert_or_error(mi, &mut feature) {
+                    return None;
+                }
+            }
+            sym::since => {
+                if !insert_or_error(mi, &mut since) {
+                    return None;
+                }
+            }
             _ => {
-                sess.emit_err(session_diagnostics::RustcAllowedUnstablePairing { span: item_sp });
+                handle_errors(
+                    &sess.parse_sess,
+                    meta.span(),
+                    AttrError::UnknownMetaItem(
+                        pprust::path_to_string(&mi.path),
+                        &["feature", "since"],
+                    ),
+                );
+                return None;
             }
         }
     }
 
-    (stab, const_stab, body_stab)
+    if let Some(s) = since && s.as_str() == VERSION_PLACEHOLDER {
+        since = Some(rust_version_symbol());
+    }
+
+    match (feature, since) {
+        (Some(feature), Some(since)) => {
+            let level = StabilityLevel::Stable { since, allowed_through_unstable_modules: false };
+            Some((feature, level))
+        }
+        (None, _) => {
+            handle_errors(&sess.parse_sess, attr.span, AttrError::MissingFeature);
+            None
+        }
+        _ => {
+            handle_errors(&sess.parse_sess, attr.span, AttrError::MissingSince);
+            None
+        }
+    }
+}
+
+/// Read the content of a `unstable`/`rustc_const_unstable`/`rustc_default_body_unstable`
+/// attribute, and return the feature name and its stability information.
+fn parse_unstability(sess: &Session, attr: &Attribute) -> Option<(Symbol, StabilityLevel)> {
+    let meta = attr.meta()?;
+    let MetaItem { kind: MetaItemKind::List(ref metas), .. } = meta else { return None };
+    let insert_or_error = |meta: &MetaItem, item: &mut Option<Symbol>| {
+        if item.is_some() {
+            handle_errors(
+                &sess.parse_sess,
+                meta.span,
+                AttrError::MultipleItem(pprust::path_to_string(&meta.path)),
+            );
+            return false;
+        }
+        if let Some(v) = meta.value_str() {
+            *item = Some(v);
+            true
+        } else {
+            sess.emit_err(session_diagnostics::IncorrectMetaItem { span: meta.span });
+            false
+        }
+    };
+
+    let mut feature = None;
+    let mut reason = None;
+    let mut issue = None;
+    let mut issue_num = None;
+    let mut is_soft = false;
+    let mut implied_by = None;
+    for meta in metas {
+        let Some(mi) = meta.meta_item() else {
+            handle_errors(
+                &sess.parse_sess,
+                meta.span(),
+                AttrError::UnsupportedLiteral(UnsupportedLiteralReason::Generic, false),
+            );
+            return None;
+        };
+
+        match mi.name_or_empty() {
+            sym::feature => {
+                if !insert_or_error(mi, &mut feature) {
+                    return None;
+                }
+            }
+            sym::reason => {
+                if !insert_or_error(mi, &mut reason) {
+                    return None;
+                }
+            }
+            sym::issue => {
+                if !insert_or_error(mi, &mut issue) {
+                    return None;
+                }
+
+                // These unwraps are safe because `insert_or_error` ensures the meta item
+                // is a name/value pair string literal.
+                issue_num = match issue.unwrap().as_str() {
+                    "none" => None,
+                    issue => match issue.parse::<NonZeroU32>() {
+                        Ok(num) => Some(num),
+                        Err(err) => {
+                            sess.emit_err(
+                                session_diagnostics::InvalidIssueString {
+                                    span: mi.span,
+                                    cause: session_diagnostics::InvalidIssueStringCause::from_int_error_kind(
+                                        mi.name_value_literal_span().unwrap(),
+                                        err.kind(),
+                                    ),
+                                },
+                            );
+                            return None;
+                        }
+                    },
+                };
+            }
+            sym::soft => {
+                if !mi.is_word() {
+                    sess.emit_err(session_diagnostics::SoftNoArgs { span: mi.span });
+                }
+                is_soft = true;
+            }
+            sym::implied_by => {
+                if !insert_or_error(mi, &mut implied_by) {
+                    return None;
+                }
+            }
+            _ => {
+                handle_errors(
+                    &sess.parse_sess,
+                    meta.span(),
+                    AttrError::UnknownMetaItem(
+                        pprust::path_to_string(&mi.path),
+                        &["feature", "reason", "issue", "soft", "implied_by"],
+                    ),
+                );
+                return None;
+            }
+        }
+    }
+
+    match (feature, reason, issue) {
+        (Some(feature), reason, Some(_)) => {
+            if !rustc_lexer::is_ident(feature.as_str()) {
+                handle_errors(&sess.parse_sess, attr.span, AttrError::NonIdentFeature);
+                return None;
+            }
+            let level = StabilityLevel::Unstable {
+                reason: UnstableReason::from_opt_reason(reason),
+                issue: issue_num,
+                is_soft,
+                implied_by,
+            };
+            Some((feature, level))
+        }
+        (None, _, _) => {
+            handle_errors(&sess.parse_sess, attr.span, AttrError::MissingFeature);
+            return None;
+        }
+        _ => {
+            sess.emit_err(session_diagnostics::MissingIssue { span: attr.span });
+            return None;
+        }
+    }
 }
 
-pub fn find_crate_name(sess: &Session, attrs: &[Attribute]) -> Option<Symbol> {
-    sess.first_attr_value_str_by_name(attrs, sym::crate_name)
+pub fn find_crate_name(attrs: &[Attribute]) -> Option<Symbol> {
+    attr::first_attr_value_str_by_name(attrs, sym::crate_name)
 }
 
 #[derive(Clone, Debug)]
@@ -1177,7 +1180,7 @@ fn allow_unstable<'a>(
     attrs: &'a [Attribute],
     symbol: Symbol,
 ) -> impl Iterator<Item = Symbol> + 'a {
-    let attrs = sess.filter_by_name(attrs, symbol);
+    let attrs = attr::filter_by_name(attrs, symbol);
     let list = attrs
         .filter_map(move |attr| {
             attr.meta_item_list().or_else(|| {
diff --git a/compiler/rustc_attr/src/lib.rs b/compiler/rustc_attr/src/lib.rs
index 5fede0a58..49818c14f 100644
--- a/compiler/rustc_attr/src/lib.rs
+++ b/compiler/rustc_attr/src/lib.rs
@@ -26,4 +26,4 @@ pub use rustc_ast::attr::*;
 
 pub(crate) use rustc_ast::HashStableContext;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_borrowck/locales/en-US.ftl b/compiler/rustc_borrowck/messages.ftl
index a3b6b5e81..a3b6b5e81 100644
--- a/compiler/rustc_borrowck/locales/en-US.ftl
+++ b/compiler/rustc_borrowck/messages.ftl
diff --git a/compiler/rustc_borrowck/src/borrow_set.rs b/compiler/rustc_borrowck/src/borrow_set.rs
index 5bb92a358..4824f6346 100644
--- a/compiler/rustc_borrowck/src/borrow_set.rs
+++ b/compiler/rustc_borrowck/src/borrow_set.rs
@@ -1,10 +1,9 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
-use crate::nll::ToRegionVid;
 use crate::path_utils::allow_two_phase_borrow;
 use crate::place_ext::PlaceExt;
 use crate::BorrowIndex;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
+use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::traversal;
 use rustc_middle::mir::visit::{MutatingUseContext, NonUseContext, PlaceContext, Visitor};
@@ -26,10 +25,10 @@ pub struct BorrowSet<'tcx> {
     /// NOTE: a given location may activate more than one borrow in the future
     /// when more general two-phase borrow support is introduced, but for now we
     /// only need to store one borrow index.
-    pub activation_map: FxHashMap<Location, Vec<BorrowIndex>>,
+    pub activation_map: FxIndexMap<Location, Vec<BorrowIndex>>,
 
     /// Map from local to all the borrows on that local.
-    pub local_map: FxHashMap<mir::Local, FxHashSet<BorrowIndex>>,
+    pub local_map: FxIndexMap<mir::Local, FxIndexSet<BorrowIndex>>,
 
     pub(crate) locals_state_at_exit: LocalsStateAtExit,
 }
@@ -175,8 +174,8 @@ struct GatherBorrows<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     body: &'a Body<'tcx>,
     location_map: FxIndexMap<Location, BorrowData<'tcx>>,
-    activation_map: FxHashMap<Location, Vec<BorrowIndex>>,
-    local_map: FxHashMap<mir::Local, FxHashSet<BorrowIndex>>,
+    activation_map: FxIndexMap<Location, Vec<BorrowIndex>>,
+    local_map: FxIndexMap<mir::Local, FxIndexSet<BorrowIndex>>,
 
     /// When we encounter a 2-phase borrow statement, it will always
     /// be assigning into a temporary TEMP:
@@ -186,7 +185,7 @@ struct GatherBorrows<'a, 'tcx> {
     /// We add TEMP into this map with `b`, where `b` is the index of
     /// the borrow. When we find a later use of this activation, we
     /// remove from the map (and add to the "tombstone" set below).
-    pending_activations: FxHashMap<mir::Local, BorrowIndex>,
+    pending_activations: FxIndexMap<mir::Local, BorrowIndex>,
 
     locals_state_at_exit: LocalsStateAtExit,
 }
@@ -204,7 +203,7 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherBorrows<'a, 'tcx> {
                 return;
             }
 
-            let region = region.to_region_vid();
+            let region = region.as_var();
 
             let borrow = BorrowData {
                 kind,
@@ -279,7 +278,7 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherBorrows<'a, 'tcx> {
             let borrow_data = &self.location_map[&location];
             assert_eq!(borrow_data.reserve_location, location);
             assert_eq!(borrow_data.kind, kind);
-            assert_eq!(borrow_data.region, region.to_region_vid());
+            assert_eq!(borrow_data.region, region.as_var());
             assert_eq!(borrow_data.borrowed_place, place);
         }
 
diff --git a/compiler/rustc_borrowck/src/constraint_generation.rs b/compiler/rustc_borrowck/src/constraint_generation.rs
index 1427f5cb3..2aa09a3f2 100644
--- a/compiler/rustc_borrowck/src/constraint_generation.rs
+++ b/compiler/rustc_borrowck/src/constraint_generation.rs
@@ -12,8 +12,8 @@ use rustc_middle::ty::visit::TypeVisitable;
 use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt};
 
 use crate::{
-    borrow_set::BorrowSet, facts::AllFacts, location::LocationTable, nll::ToRegionVid,
-    places_conflict, region_infer::values::LivenessValues,
+    borrow_set::BorrowSet, facts::AllFacts, location::LocationTable, places_conflict,
+    region_infer::values::LivenessValues,
 };
 
 pub(super) fn generate_constraints<'tcx>(
@@ -170,7 +170,7 @@ impl<'cx, 'tcx> ConstraintGeneration<'cx, 'tcx> {
         debug!("add_regular_live_constraint(live_ty={:?}, location={:?})", live_ty, location);
 
         self.infcx.tcx.for_each_free_region(&live_ty, |live_region| {
-            let vid = live_region.to_region_vid();
+            let vid = live_region.as_var();
             self.liveness_constraints.add_element(vid, location);
         });
     }
diff --git a/compiler/rustc_borrowck/src/constraints/graph.rs b/compiler/rustc_borrowck/src/constraints/graph.rs
index c780d0479..f5a34cb05 100644
--- a/compiler/rustc_borrowck/src/constraints/graph.rs
+++ b/compiler/rustc_borrowck/src/constraints/graph.rs
@@ -13,7 +13,7 @@ use crate::{
 /// The construct graph organizes the constraints by their end-points.
 /// It can be used to view a `R1: R2` constraint as either an edge `R1
 /// -> R2` or `R2 -> R1` depending on the direction type `D`.
-pub(crate) struct ConstraintGraph<D: ConstraintGraphDirecton> {
+pub(crate) struct ConstraintGraph<D: ConstraintGraphDirection> {
     _direction: D,
     first_constraints: IndexVec<RegionVid, Option<OutlivesConstraintIndex>>,
     next_constraints: IndexVec<OutlivesConstraintIndex, Option<OutlivesConstraintIndex>>,
@@ -25,7 +25,7 @@ pub(crate) type ReverseConstraintGraph = ConstraintGraph<Reverse>;
 
 /// Marker trait that controls whether a `R1: R2` constraint
 /// represents an edge `R1 -> R2` or `R2 -> R1`.
-pub(crate) trait ConstraintGraphDirecton: Copy + 'static {
+pub(crate) trait ConstraintGraphDirection: Copy + 'static {
     fn start_region(c: &OutlivesConstraint<'_>) -> RegionVid;
     fn end_region(c: &OutlivesConstraint<'_>) -> RegionVid;
     fn is_normal() -> bool;
@@ -38,7 +38,7 @@ pub(crate) trait ConstraintGraphDirecton: Copy + 'static {
 #[derive(Copy, Clone, Debug)]
 pub(crate) struct Normal;
 
-impl ConstraintGraphDirecton for Normal {
+impl ConstraintGraphDirection for Normal {
     fn start_region(c: &OutlivesConstraint<'_>) -> RegionVid {
         c.sup
     }
@@ -59,7 +59,7 @@ impl ConstraintGraphDirecton for Normal {
 #[derive(Copy, Clone, Debug)]
 pub(crate) struct Reverse;
 
-impl ConstraintGraphDirecton for Reverse {
+impl ConstraintGraphDirection for Reverse {
     fn start_region(c: &OutlivesConstraint<'_>) -> RegionVid {
         c.sub
     }
@@ -73,7 +73,7 @@ impl ConstraintGraphDirecton for Reverse {
     }
 }
 
-impl<D: ConstraintGraphDirecton> ConstraintGraph<D> {
+impl<D: ConstraintGraphDirection> ConstraintGraph<D> {
     /// Creates a "dependency graph" where each region constraint `R1:
     /// R2` is treated as an edge `R1 -> R2`. We use this graph to
     /// construct SCCs for region inference but also for error
@@ -133,7 +133,7 @@ impl<D: ConstraintGraphDirecton> ConstraintGraph<D> {
     }
 }
 
-pub(crate) struct Edges<'s, 'tcx, D: ConstraintGraphDirecton> {
+pub(crate) struct Edges<'s, 'tcx, D: ConstraintGraphDirection> {
     graph: &'s ConstraintGraph<D>,
     constraints: &'s OutlivesConstraintSet<'tcx>,
     pointer: Option<OutlivesConstraintIndex>,
@@ -141,7 +141,7 @@ pub(crate) struct Edges<'s, 'tcx, D: ConstraintGraphDirecton> {
     static_region: RegionVid,
 }
 
-impl<'s, 'tcx, D: ConstraintGraphDirecton> Iterator for Edges<'s, 'tcx, D> {
+impl<'s, 'tcx, D: ConstraintGraphDirection> Iterator for Edges<'s, 'tcx, D> {
     type Item = OutlivesConstraint<'tcx>;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -174,13 +174,13 @@ impl<'s, 'tcx, D: ConstraintGraphDirecton> Iterator for Edges<'s, 'tcx, D> {
 /// This struct brings together a constraint set and a (normal, not
 /// reverse) constraint graph. It implements the graph traits and is
 /// usd for doing the SCC computation.
-pub(crate) struct RegionGraph<'s, 'tcx, D: ConstraintGraphDirecton> {
+pub(crate) struct RegionGraph<'s, 'tcx, D: ConstraintGraphDirection> {
     set: &'s OutlivesConstraintSet<'tcx>,
     constraint_graph: &'s ConstraintGraph<D>,
     static_region: RegionVid,
 }
 
-impl<'s, 'tcx, D: ConstraintGraphDirecton> RegionGraph<'s, 'tcx, D> {
+impl<'s, 'tcx, D: ConstraintGraphDirection> RegionGraph<'s, 'tcx, D> {
     /// Creates a "dependency graph" where each region constraint `R1:
     /// R2` is treated as an edge `R1 -> R2`. We use this graph to
     /// construct SCCs for region inference but also for error
@@ -202,11 +202,11 @@ impl<'s, 'tcx, D: ConstraintGraphDirecton> RegionGraph<'s, 'tcx, D> {
     }
 }
 
-pub(crate) struct Successors<'s, 'tcx, D: ConstraintGraphDirecton> {
+pub(crate) struct Successors<'s, 'tcx, D: ConstraintGraphDirection> {
     edges: Edges<'s, 'tcx, D>,
 }
 
-impl<'s, 'tcx, D: ConstraintGraphDirecton> Iterator for Successors<'s, 'tcx, D> {
+impl<'s, 'tcx, D: ConstraintGraphDirection> Iterator for Successors<'s, 'tcx, D> {
     type Item = RegionVid;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -214,23 +214,25 @@ impl<'s, 'tcx, D: ConstraintGraphDirecton> Iterator for Successors<'s, 'tcx, D>
     }
 }
 
-impl<'s, 'tcx, D: ConstraintGraphDirecton> graph::DirectedGraph for RegionGraph<'s, 'tcx, D> {
+impl<'s, 'tcx, D: ConstraintGraphDirection> graph::DirectedGraph for RegionGraph<'s, 'tcx, D> {
     type Node = RegionVid;
 }
 
-impl<'s, 'tcx, D: ConstraintGraphDirecton> graph::WithNumNodes for RegionGraph<'s, 'tcx, D> {
+impl<'s, 'tcx, D: ConstraintGraphDirection> graph::WithNumNodes for RegionGraph<'s, 'tcx, D> {
     fn num_nodes(&self) -> usize {
         self.constraint_graph.first_constraints.len()
     }
 }
 
-impl<'s, 'tcx, D: ConstraintGraphDirecton> graph::WithSuccessors for RegionGraph<'s, 'tcx, D> {
+impl<'s, 'tcx, D: ConstraintGraphDirection> graph::WithSuccessors for RegionGraph<'s, 'tcx, D> {
     fn successors(&self, node: Self::Node) -> <Self as graph::GraphSuccessors<'_>>::Iter {
         self.outgoing_regions(node)
     }
 }
 
-impl<'s, 'tcx, D: ConstraintGraphDirecton> graph::GraphSuccessors<'_> for RegionGraph<'s, 'tcx, D> {
+impl<'s, 'tcx, D: ConstraintGraphDirection> graph::GraphSuccessors<'_>
+    for RegionGraph<'s, 'tcx, D>
+{
     type Item = RegionVid;
     type Iter = Successors<'s, 'tcx, D>;
 }
diff --git a/compiler/rustc_borrowck/src/constraints/mod.rs b/compiler/rustc_borrowck/src/constraints/mod.rs
index f370c0216..d2d9779db 100644
--- a/compiler/rustc_borrowck/src/constraints/mod.rs
+++ b/compiler/rustc_borrowck/src/constraints/mod.rs
@@ -2,7 +2,7 @@
 #![deny(rustc::diagnostic_outside_of_impl)]
 
 use rustc_data_structures::graph::scc::Sccs;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::ty::{RegionVid, VarianceDiagInfo};
 use rustc_span::Span;
@@ -60,7 +60,9 @@ impl<'tcx> OutlivesConstraintSet<'tcx> {
         Sccs::new(region_graph)
     }
 
-    pub(crate) fn outlives(&self) -> &IndexVec<OutlivesConstraintIndex, OutlivesConstraint<'tcx>> {
+    pub(crate) fn outlives(
+        &self,
+    ) -> &IndexSlice<OutlivesConstraintIndex, OutlivesConstraint<'tcx>> {
         &self.outlives
     }
 }
diff --git a/compiler/rustc_borrowck/src/consumers.rs b/compiler/rustc_borrowck/src/consumers.rs
index 055b281bc..cb1a65222 100644
--- a/compiler/rustc_borrowck/src/consumers.rs
+++ b/compiler/rustc_borrowck/src/consumers.rs
@@ -3,7 +3,7 @@
 //! This file provides API for compiler consumers.
 
 use rustc_hir::def_id::LocalDefId;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::IndexSlice;
 use rustc_infer::infer::{DefiningAnchor, TyCtxtInferExt};
 use rustc_middle::mir::Body;
 use rustc_middle::ty::{self, TyCtxt};
@@ -35,6 +35,6 @@ pub fn get_body_with_borrowck_facts(
     let (input_body, promoted) = tcx.mir_promoted(def);
     let infcx = tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bind(def.did)).build();
     let input_body: &Body<'_> = &input_body.borrow();
-    let promoted: &IndexVec<_, _> = &promoted.borrow();
+    let promoted: &IndexSlice<_, _> = &promoted.borrow();
     *super::do_mir_borrowck(&infcx, input_body, promoted, true).1.unwrap()
 }
diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs
index 2821677c5..94939c7e4 100644
--- a/compiler/rustc_borrowck/src/dataflow.rs
+++ b/compiler/rustc_borrowck/src/dataflow.rs
@@ -1,6 +1,6 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::{self, BasicBlock, Body, Location, Place};
 use rustc_middle::ty::RegionVid;
@@ -11,9 +11,7 @@ use rustc_mir_dataflow::{self, fmt::DebugWithContext, CallReturnPlaces, GenKill}
 use rustc_mir_dataflow::{Analysis, Direction, Results};
 use std::fmt;
 
-use crate::{
-    places_conflict, BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext, ToRegionVid,
-};
+use crate::{places_conflict, BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext};
 
 /// A tuple with named fields that can hold either the results or the transient state of the
 /// dataflow analyses used by the borrow checker.
@@ -124,7 +122,7 @@ pub struct Borrows<'a, 'tcx> {
     body: &'a Body<'tcx>,
 
     borrow_set: &'a BorrowSet<'tcx>,
-    borrows_out_of_scope_at_location: FxHashMap<Location, Vec<BorrowIndex>>,
+    borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
 struct StackEntry {
@@ -138,7 +136,7 @@ struct OutOfScopePrecomputer<'a, 'tcx> {
     visit_stack: Vec<StackEntry>,
     body: &'a Body<'tcx>,
     regioncx: &'a RegionInferenceContext<'tcx>,
-    borrows_out_of_scope_at_location: FxHashMap<Location, Vec<BorrowIndex>>,
+    borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
 impl<'a, 'tcx> OutOfScopePrecomputer<'a, 'tcx> {
@@ -148,7 +146,7 @@ impl<'a, 'tcx> OutOfScopePrecomputer<'a, 'tcx> {
             visit_stack: vec![],
             body,
             regioncx,
-            borrows_out_of_scope_at_location: FxHashMap::default(),
+            borrows_out_of_scope_at_location: FxIndexMap::default(),
         }
     }
 }
@@ -242,7 +240,7 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> {
     ) -> Self {
         let mut prec = OutOfScopePrecomputer::new(body, nonlexical_regioncx);
         for (borrow_index, borrow_data) in borrow_set.iter_enumerated() {
-            let borrow_region = borrow_data.region.to_region_vid();
+            let borrow_region = borrow_data.region;
             let location = borrow_data.reserve_location;
 
             prec.precompute_borrows_out_of_scope(borrow_index, borrow_region, location);
@@ -306,7 +304,7 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> {
         }
 
         // By passing `PlaceConflictBias::NoOverlap`, we conservatively assume that any given
-        // pair of array indices are unequal, so that when `places_conflict` returns true, we
+        // pair of array indices are not equal, so that when `places_conflict` returns true, we
         // will be assured that two places being compared definitely denotes the same sets of
         // locations.
         let definitely_conflicting_borrows = other_borrows_of_local.filter(|&i| {
@@ -390,6 +388,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
             | mir::StatementKind::Deinit(..)
             | mir::StatementKind::StorageLive(..)
             | mir::StatementKind::Retag { .. }
+            | mir::StatementKind::PlaceMention(..)
             | mir::StatementKind::AscribeUserType(..)
             | mir::StatementKind::Coverage(..)
             | mir::StatementKind::Intrinsic(..)
diff --git a/compiler/rustc_borrowck/src/def_use.rs b/compiler/rustc_borrowck/src/def_use.rs
index 8e62a0198..6deb4e361 100644
--- a/compiler/rustc_borrowck/src/def_use.rs
+++ b/compiler/rustc_borrowck/src/def_use.rs
@@ -50,7 +50,6 @@ pub fn categorize(context: PlaceContext) -> Option<DefUse> {
         PlaceContext::MutatingUse(MutatingUseContext::Borrow) |
         PlaceContext::NonMutatingUse(NonMutatingUseContext::SharedBorrow) |
         PlaceContext::NonMutatingUse(NonMutatingUseContext::ShallowBorrow) |
-        PlaceContext::NonMutatingUse(NonMutatingUseContext::UniqueBorrow) |
 
         PlaceContext::MutatingUse(MutatingUseContext::AddressOf) |
         PlaceContext::NonMutatingUse(NonMutatingUseContext::AddressOf) |
@@ -72,6 +71,8 @@ pub fn categorize(context: PlaceContext) -> Option<DefUse> {
         PlaceContext::MutatingUse(MutatingUseContext::Drop) =>
             Some(DefUse::Drop),
 
+        // This statement exists to help unsafeck. It does not require the place to be live.
+        PlaceContext::NonUse(NonUseContext::PlaceMention) => None,
         // Debug info is neither def nor use.
         PlaceContext::NonUse(NonUseContext::VarDebugInfo) => None,
 
diff --git a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
index 68205fa45..84f75caa6 100644
--- a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
@@ -181,8 +181,8 @@ trait TypeOpInfo<'tcx> {
         };
 
         let placeholder_region = tcx.mk_re_placeholder(ty::Placeholder {
-            name: placeholder.name,
             universe: adjusted_universe.into(),
+            bound: placeholder.bound,
         });
 
         let error_region =
@@ -191,8 +191,8 @@ trait TypeOpInfo<'tcx> {
                     error_placeholder.universe.as_u32().checked_sub(base_universe.as_u32());
                 adjusted_universe.map(|adjusted| {
                     tcx.mk_re_placeholder(ty::Placeholder {
-                        name: error_placeholder.name,
                         universe: adjusted.into(),
+                        bound: error_placeholder.bound,
                     })
                 })
             } else {
diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index cb97699d7..75a3dd0c0 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -1,7 +1,7 @@
 use either::Either;
 use rustc_const_eval::util::CallKind;
 use rustc_data_structures::captures::Captures;
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::{
     struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, MultiSpan,
 };
@@ -24,6 +24,7 @@ use rustc_span::hygiene::DesugaringKind;
 use rustc_span::symbol::{kw, sym};
 use rustc_span::{BytePos, Span, Symbol};
 use rustc_trait_selection::infer::InferCtxtExt;
+use rustc_trait_selection::traits::ObligationCtxt;
 
 use crate::borrow_set::TwoPhaseActivation;
 use crate::borrowck_errors;
@@ -173,7 +174,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
             let mut is_loop_move = false;
             let mut in_pattern = false;
-            let mut seen_spans = FxHashSet::default();
+            let mut seen_spans = FxIndexSet::default();
 
             for move_site in &move_site_vec {
                 let move_out = self.move_data.moves[(*move_site).moi];
@@ -760,20 +761,12 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         else { return; };
         // Try to find predicates on *generic params* that would allow copying `ty`
         let infcx = tcx.infer_ctxt().build();
-        let copy_did = infcx.tcx.require_lang_item(LangItem::Copy, Some(span));
-        let cause = ObligationCause::new(
-            span,
-            self.mir_def_id(),
-            rustc_infer::traits::ObligationCauseCode::MiscObligation,
-        );
-        let errors = rustc_trait_selection::traits::fully_solve_bound(
-            &infcx,
-            cause,
-            self.param_env,
-            // Erase any region vids from the type, which may not be resolved
-            infcx.tcx.erase_regions(ty),
-            copy_did,
-        );
+        let ocx = ObligationCtxt::new(&infcx);
+        let copy_did = tcx.require_lang_item(LangItem::Copy, Some(span));
+        let cause = ObligationCause::misc(span, self.mir_def_id());
+
+        ocx.register_bound(cause, self.param_env, infcx.tcx.erase_regions(ty), copy_did);
+        let errors = ocx.select_all_or_error();
 
         // Only emit suggestion if all required predicates are on generic
         let predicates: Result<Vec<_>, _> = errors
@@ -1467,6 +1460,32 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
     /// Reports StorageDeadOrDrop of `place` conflicts with `borrow`.
     ///
+    /// Depending on the origin of the StorageDeadOrDrop, this may be
+    /// reported as either a drop or an illegal mutation of a borrowed value.
+    /// The latter is preferred when the this is a drop triggered by a
+    /// reassignment, as it's more user friendly to report a problem with the
+    /// explicit assignment than the implicit drop.
+    #[instrument(level = "debug", skip(self))]
+    pub(crate) fn report_storage_dead_or_drop_of_borrowed(
+        &mut self,
+        location: Location,
+        place_span: (Place<'tcx>, Span),
+        borrow: &BorrowData<'tcx>,
+    ) {
+        // It's sufficient to check the last desugaring as Replace is the last
+        // one to be applied.
+        if let Some(DesugaringKind::Replace) = place_span.1.desugaring_kind() {
+            self.report_illegal_mutation_of_borrowed(location, place_span, borrow)
+        } else {
+            self.report_borrowed_value_does_not_live_long_enough(
+                location,
+                borrow,
+                place_span,
+                Some(WriteKind::StorageDeadOrDrop),
+            )
+        }
+    }
+
     /// This means that some data referenced by `borrow` needs to live
     /// past the point where the StorageDeadOrDrop of `place` occurs.
     /// This is usually interpreted as meaning that `place` has too
@@ -1959,16 +1978,18 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let (place_desc, note) = if let Some(place_desc) = opt_place_desc {
             let local_kind = if let Some(local) = borrow.borrowed_place.as_local() {
                 match self.body.local_kind(local) {
-                    LocalKind::ReturnPointer | LocalKind::Temp => {
-                        bug!("temporary or return pointer with a name")
+                    LocalKind::Temp if self.body.local_decls[local].is_user_variable() => {
+                        "local variable "
                     }
-                    LocalKind::Var => "local variable ",
                     LocalKind::Arg
                         if !self.upvars.is_empty() && local == ty::CAPTURE_STRUCT_LOCAL =>
                     {
                         "variable captured by `move` "
                     }
                     LocalKind::Arg => "function parameter ",
+                    LocalKind::ReturnPointer | LocalKind::Temp => {
+                        bug!("temporary or return pointer with a name")
+                    }
                 }
             } else {
                 "local data "
@@ -1982,16 +2003,16 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 self.prefixes(borrow.borrowed_place.as_ref(), PrefixSet::All).last().unwrap();
             let local = root_place.local;
             match self.body.local_kind(local) {
-                LocalKind::ReturnPointer | LocalKind::Temp => {
-                    ("temporary value".to_string(), "temporary value created here".to_string())
-                }
                 LocalKind::Arg => (
                     "function parameter".to_string(),
                     "function parameter borrowed here".to_string(),
                 ),
-                LocalKind::Var => {
+                LocalKind::Temp if self.body.local_decls[local].is_user_variable() => {
                     ("local binding".to_string(), "local binding introduced here".to_string())
                 }
+                LocalKind::ReturnPointer | LocalKind::Temp => {
+                    ("temporary value".to_string(), "temporary value created here".to_string())
+                }
             }
         };
 
@@ -2197,8 +2218,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             }
         }
 
-        let mut visited = FxHashSet::default();
-        let mut move_locations = FxHashSet::default();
+        let mut visited = FxIndexSet::default();
+        let mut move_locations = FxIndexSet::default();
         let mut reinits = vec![];
         let mut result = vec![];
 
@@ -2325,7 +2346,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let reinits_reachable = reinits
             .into_iter()
             .filter(|reinit| {
-                let mut visited = FxHashSet::default();
+                let mut visited = FxIndexSet::default();
                 let mut stack = vec![*reinit];
                 while let Some(location) = stack.pop() {
                     if !visited.insert(location) {
@@ -2456,15 +2477,14 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let (place_description, assigned_span) = match local_decl {
             Some(LocalDecl {
                 local_info:
-                    Some(box LocalInfo::User(
-                        ClearCrossCrate::Clear
-                        | ClearCrossCrate::Set(BindingForm::Var(VarBindingForm {
+                    ClearCrossCrate::Set(
+                        box LocalInfo::User(BindingForm::Var(VarBindingForm {
                             opt_match_place: None,
                             ..
-                        })),
-                    ))
-                    | Some(box LocalInfo::StaticRef { .. })
-                    | None,
+                        }))
+                        | box LocalInfo::StaticRef { .. }
+                        | box LocalInfo::Boring,
+                    ),
                 ..
             })
             | None => (self.describe_any_place(place.as_ref()), assigned_span),
diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
index 19855075c..8860395e7 100644
--- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
@@ -3,11 +3,11 @@
 use rustc_errors::{Applicability, Diagnostic};
 use rustc_hir as hir;
 use rustc_hir::intravisit::Visitor;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::IndexSlice;
 use rustc_infer::infer::NllRegionVariableOrigin;
 use rustc_middle::mir::{
-    Body, CastKind, ConstraintCategory, FakeReadCause, Local, Location, Operand, Place, Rvalue,
-    Statement, StatementKind, TerminatorKind,
+    Body, CastKind, ConstraintCategory, FakeReadCause, Local, LocalInfo, Location, Operand, Place,
+    Rvalue, Statement, StatementKind, TerminatorKind,
 };
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::{self, RegionVid, TyCtxt};
@@ -60,7 +60,7 @@ impl<'tcx> BorrowExplanation<'tcx> {
         &self,
         tcx: TyCtxt<'tcx>,
         body: &Body<'tcx>,
-        local_names: &IndexVec<Local, Option<Symbol>>,
+        local_names: &IndexSlice<Local, Option<Symbol>>,
         err: &mut Diagnostic,
         borrow_desc: &str,
         borrow_span: Option<Span>,
@@ -220,7 +220,7 @@ impl<'tcx> BorrowExplanation<'tcx> {
                         );
                         err.span_label(body.source_info(drop_loc).span, message);
 
-                        if let Some(info) = &local_decl.is_block_tail {
+                        if let LocalInfo::BlockTailTemp(info) = local_decl.local_info() {
                             if info.tail_result_is_ignored {
                                 // #85581: If the first mutable borrow's scope contains
                                 // the second borrow, this suggestion isn't helpful.
diff --git a/compiler/rustc_borrowck/src/diagnostics/find_use.rs b/compiler/rustc_borrowck/src/diagnostics/find_use.rs
index 15f42e26c..2495613fe 100644
--- a/compiler/rustc_borrowck/src/diagnostics/find_use.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/find_use.rs
@@ -6,12 +6,11 @@ use std::rc::Rc;
 
 use crate::{
     def_use::{self, DefUse},
-    nll::ToRegionVid,
     region_infer::{Cause, RegionInferenceContext},
 };
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_middle::mir::visit::{MirVisitable, PlaceContext, Visitor};
-use rustc_middle::mir::{Body, Local, Location};
+use rustc_middle::mir::{self, Body, Local, Location};
 use rustc_middle::ty::{RegionVid, TyCtxt};
 
 pub(crate) fn find<'tcx>(
@@ -37,7 +36,7 @@ struct UseFinder<'cx, 'tcx> {
 impl<'cx, 'tcx> UseFinder<'cx, 'tcx> {
     fn find(&mut self) -> Option<Cause> {
         let mut queue = VecDeque::new();
-        let mut visited = FxHashSet::default();
+        let mut visited = FxIndexSet::default();
 
         queue.push_back(self.start_point);
         while let Some(p) = queue.pop_front() {
@@ -70,7 +69,10 @@ impl<'cx, 'tcx> UseFinder<'cx, 'tcx> {
                             block_data
                                 .terminator()
                                 .successors()
-                                .filter(|&bb| Some(&Some(bb)) != block_data.terminator().unwind())
+                                .filter(|&bb| {
+                                    Some(&mir::UnwindAction::Cleanup(bb))
+                                        != block_data.terminator().unwind()
+                                })
                                 .map(|bb| Location { statement_index: 0, block: bb }),
                         );
                     }
@@ -114,7 +116,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for DefUseVisitor<'cx, 'tcx> {
 
         let mut found_it = false;
         self.tcx.for_each_free_region(&local_ty, |r| {
-            if r.to_region_vid() == self.region_vid {
+            if r.as_var() == self.region_vid {
                 found_it = true;
             }
         });
diff --git a/compiler/rustc_borrowck/src/diagnostics/mod.rs b/compiler/rustc_borrowck/src/diagnostics/mod.rs
index a99fd594a..110354a20 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mod.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mod.rs
@@ -6,18 +6,19 @@ use rustc_errors::{Applicability, Diagnostic};
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, Namespace};
 use rustc_hir::GeneratorKind;
+use rustc_index::vec::IndexSlice;
 use rustc_infer::infer::{LateBoundRegionConversionTime, TyCtxtInferExt};
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::{
-    AggregateKind, Constant, FakeReadCause, Field, Local, LocalInfo, LocalKind, Location, Operand,
-    Place, PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
+    AggregateKind, Constant, FakeReadCause, Local, LocalInfo, LocalKind, Location, Operand, Place,
+    PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
 };
 use rustc_middle::ty::print::Print;
-use rustc_middle::ty::{self, DefIdTree, Instance, Ty, TyCtxt};
+use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
 use rustc_mir_dataflow::move_paths::{InitLocation, LookupResult};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{symbol::sym, Span, Symbol, DUMMY_SP};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
 use rustc_trait_selection::traits::{
     type_known_to_meet_bound_modulo_regions, Obligation, ObligationCause,
@@ -196,10 +197,10 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         if self.body.local_decls[local].is_ref_for_guard() {
                             continue;
                         }
-                        if let Some(box LocalInfo::StaticRef { def_id, .. }) =
-                            &self.body.local_decls[local].local_info
+                        if let LocalInfo::StaticRef { def_id, .. } =
+                            *self.body.local_decls[local].local_info()
                         {
-                            buf.push_str(self.infcx.tcx.item_name(*def_id).as_str());
+                            buf.push_str(self.infcx.tcx.item_name(def_id).as_str());
                             ok = Ok(());
                             continue;
                         }
@@ -302,7 +303,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
     fn describe_field(
         &self,
         place: PlaceRef<'tcx>,
-        field: Field,
+        field: FieldIdx,
         including_tuple_field: IncludingTupleField,
     ) -> Option<String> {
         let place_ty = match place {
@@ -331,7 +332,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
     fn describe_field_from_ty(
         &self,
         ty: Ty<'_>,
-        field: Field,
+        field: FieldIdx,
         variant_index: Option<VariantIdx>,
         including_tuple_field: IncludingTupleField,
     ) -> Option<String> {
@@ -350,7 +351,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     if !including_tuple_field.0 && variant.ctor_kind() == Some(CtorKind::Fn) {
                         return None;
                     }
-                    Some(variant.fields[field.index()].name.to_string())
+                    Some(variant.fields[field].name.to_string())
                 }
                 ty::Tuple(_) => Some(field.index().to_string()),
                 ty::Ref(_, ty, _) | ty::RawPtr(ty::TypeAndMut { ty, .. }) => {
@@ -466,9 +467,10 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         if let ty::Ref(region, ..) = ty.kind() {
             match **region {
                 ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
-                | ty::RePlaceholder(ty::PlaceholderRegion { name: br, .. }) => {
-                    printer.region_highlight_mode.highlighting_bound_region(br, counter)
-                }
+                | ty::RePlaceholder(ty::PlaceholderRegion {
+                    bound: ty::BoundRegion { kind: br, .. },
+                    ..
+                }) => printer.region_highlight_mode.highlighting_bound_region(br, counter),
                 _ => {}
             }
         }
@@ -484,9 +486,10 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let region = if let ty::Ref(region, ..) = ty.kind() {
             match **region {
                 ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
-                | ty::RePlaceholder(ty::PlaceholderRegion { name: br, .. }) => {
-                    printer.region_highlight_mode.highlighting_bound_region(br, counter)
-                }
+                | ty::RePlaceholder(ty::PlaceholderRegion {
+                    bound: ty::BoundRegion { kind: br, .. },
+                    ..
+                }) => printer.region_highlight_mode.highlighting_bound_region(br, counter),
                 _ => {}
             }
             region
@@ -825,7 +828,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     debug!("move_spans: def_id={:?} place={:?}", closure_def_id, place);
                     let places = &[Operand::Move(place)];
                     if let Some((args_span, generator_kind, capture_kind_span, path_span)) =
-                        self.closure_span(closure_def_id, moved_place, places)
+                        self.closure_span(closure_def_id, moved_place, IndexSlice::from_raw(places))
                     {
                         return ClosureUse {
                             generator_kind,
@@ -925,7 +928,20 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             return OtherUse(use_span);
         }
 
-        for stmt in &self.body[location.block].statements[location.statement_index + 1..] {
+        // drop and replace might have moved the assignment to the next block
+        let maybe_additional_statement =
+            if let TerminatorKind::Drop { target: drop_target, .. } =
+                self.body[location.block].terminator().kind
+            {
+                self.body[drop_target].statements.first()
+            } else {
+                None
+            };
+
+        let statements =
+            self.body[location.block].statements[location.statement_index + 1..].iter();
+
+        for stmt in statements.chain(maybe_additional_statement) {
             if let StatementKind::Assign(box (_, Rvalue::Aggregate(kind, places))) = &stmt.kind {
                 let (&def_id, is_generator) = match kind {
                     box AggregateKind::Closure(def_id, _) => (def_id, false),
@@ -962,7 +978,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         &self,
         def_id: LocalDefId,
         target_place: PlaceRef<'tcx>,
-        places: &[Operand<'tcx>],
+        places: &IndexSlice<FieldIdx, Operand<'tcx>>,
     ) -> Option<(Span, Option<GeneratorKind>, Span, Span)> {
         debug!(
             "closure_span: def_id={:?} target_place={:?} places={:?}",
@@ -1065,7 +1081,6 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                                     self.param_env,
                                     tcx.mk_imm_ref(tcx.lifetimes.re_erased, tcx.erase_regions(ty)),
                                     def_id,
-                                    DUMMY_SP,
                                 )
                             }
                             _ => false,
diff --git a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
index 5e4c7292e..3662bec0c 100644
--- a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
@@ -102,14 +102,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                         //
                         // opt_match_place is None for let [mut] x = ... statements,
                         // whether or not the right-hand side is a place expression
-                        if let Some(box LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
-                            VarBindingForm {
-                                opt_match_place: Some((opt_match_place, match_span)),
-                                binding_mode: _,
-                                opt_ty_info: _,
-                                pat_span: _,
-                            },
-                        )))) = local_decl.local_info
+                        if let LocalInfo::User(BindingForm::Var(VarBindingForm {
+                            opt_match_place: Some((opt_match_place, match_span)),
+                            binding_mode: _,
+                            opt_ty_info: _,
+                            pat_span: _,
+                        })) = *local_decl.local_info()
                         {
                             let stmt_source_info = self.body.source_info(location);
                             self.append_binding_error(
@@ -478,9 +476,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         let mut suggestions: Vec<(Span, String, String)> = Vec::new();
         for local in binds_to {
             let bind_to = &self.body.local_decls[*local];
-            if let Some(box LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
-                VarBindingForm { pat_span, .. },
-            )))) = bind_to.local_info
+            if let LocalInfo::User(BindingForm::Var(VarBindingForm { pat_span, .. })) =
+                *bind_to.local_info()
             {
                 let Ok(pat_snippet) =
                     self.infcx.tcx.sess.source_map().span_to_snippet(pat_span) else { continue; };
diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index 328ac880d..9d9040096 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -7,11 +7,12 @@ use rustc_middle::mir::{Mutability, Place, PlaceRef, ProjectionElem};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::{
     hir::place::PlaceBase,
-    mir::{self, BindingForm, ClearCrossCrate, Local, LocalDecl, LocalInfo, LocalKind, Location},
+    mir::{self, BindingForm, Local, LocalDecl, LocalInfo, LocalKind, Location},
 };
 use rustc_span::source_map::DesugaringKind;
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{sym, BytePos, Span};
+use rustc_target::abi::FieldIdx;
 
 use crate::diagnostics::BorrowedContentSource;
 use crate::MirBorrowckCtxt;
@@ -105,8 +106,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     reason = String::new();
                 } else {
                     item_msg = access_place_desc;
-                    let local_info = &self.body.local_decls[local].local_info;
-                    if let Some(box LocalInfo::StaticRef { def_id, .. }) = *local_info {
+                    let local_info = self.body.local_decls[local].local_info();
+                    if let LocalInfo::StaticRef { def_id, .. } = *local_info {
                         let static_name = &self.infcx.tcx.item_name(def_id);
                         reason = format!(", as `{static_name}` is an immutable static item");
                     } else {
@@ -120,9 +121,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     && !self.upvars.is_empty()
                 {
                     item_msg = access_place_desc;
-                    debug_assert!(
-                        self.body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty.is_region_ptr()
-                    );
+                    debug_assert!(self.body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty.is_ref());
                     debug_assert!(is_closure_or_generator(
                         Place::ty_from(
                             the_place_err.local,
@@ -305,15 +304,13 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     ..
                 }) = &self.body[location.block].statements.get(location.statement_index)
                 {
-                    match decl.local_info {
-                        Some(box LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
-                            mir::VarBindingForm {
-                                binding_mode: ty::BindingMode::BindByValue(Mutability::Not),
-                                opt_ty_info: Some(sp),
-                                opt_match_place: _,
-                                pat_span: _,
-                            },
-                        )))) => {
+                    match *decl.local_info() {
+                        LocalInfo::User(BindingForm::Var(mir::VarBindingForm {
+                            binding_mode: ty::BindingMode::BindByValue(Mutability::Not),
+                            opt_ty_info: Some(sp),
+                            opt_match_place: _,
+                            pat_span: _,
+                        })) => {
                             if suggest {
                                 err.span_note(sp, "the binding is already a mutable borrow");
                             }
@@ -346,10 +343,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     }
                 } else if decl.mutability.is_not() {
                     if matches!(
-                        decl.local_info,
-                        Some(box LocalInfo::User(ClearCrossCrate::Set(BindingForm::ImplicitSelf(
-                            hir::ImplicitSelfKind::MutRef
-                        ),)))
+                        decl.local_info(),
+                        LocalInfo::User(BindingForm::ImplicitSelf(hir::ImplicitSelfKind::MutRef))
                     ) {
                         err.note(
                             "as `Self` may be unsized, this call attempts to take `&mut &mut self`",
@@ -474,30 +469,23 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             {
                 let local_decl = &self.body.local_decls[local];
 
-                let (pointer_sigil, pointer_desc) = if local_decl.ty.is_region_ptr() {
-                    ("&", "reference")
-                } else {
-                    ("*const", "pointer")
-                };
+                let (pointer_sigil, pointer_desc) =
+                    if local_decl.ty.is_ref() { ("&", "reference") } else { ("*const", "pointer") };
 
                 match self.local_names[local] {
                     Some(name) if !local_decl.from_compiler_desugaring() => {
-                        let label = match local_decl.local_info.as_deref().unwrap() {
-                            LocalInfo::User(ClearCrossCrate::Set(
-                                mir::BindingForm::ImplicitSelf(_),
-                            )) => {
+                        let label = match *local_decl.local_info() {
+                            LocalInfo::User(mir::BindingForm::ImplicitSelf(_)) => {
                                 let (span, suggestion) =
                                     suggest_ampmut_self(self.infcx.tcx, local_decl);
                                 Some((true, span, suggestion))
                             }
 
-                            LocalInfo::User(ClearCrossCrate::Set(mir::BindingForm::Var(
-                                mir::VarBindingForm {
-                                    binding_mode: ty::BindingMode::BindByValue(_),
-                                    opt_ty_info,
-                                    ..
-                                },
-                            ))) => {
+                            LocalInfo::User(mir::BindingForm::Var(mir::VarBindingForm {
+                                binding_mode: ty::BindingMode::BindByValue(_),
+                                opt_ty_info,
+                                ..
+                            })) => {
                                 // check if the RHS is from desugaring
                                 let opt_assignment_rhs_span =
                                     self.body.find_assignments(local).first().map(|&location| {
@@ -534,16 +522,15 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                                                 self.infcx.tcx,
                                                 local_decl,
                                                 opt_assignment_rhs_span,
-                                                *opt_ty_info,
+                                                opt_ty_info,
                                             )
                                         } else {
-                                            match local_decl.local_info.as_deref() {
-                                                Some(LocalInfo::User(ClearCrossCrate::Set(
-                                                    mir::BindingForm::Var(mir::VarBindingForm {
-                                                        opt_ty_info: None,
-                                                        ..
-                                                    }),
-                                                ))) => {
+                                            match local_decl.local_info() {
+                                                LocalInfo::User(mir::BindingForm::Var(
+                                                    mir::VarBindingForm {
+                                                        opt_ty_info: None, ..
+                                                    },
+                                                )) => {
                                                     let (span, sugg) = suggest_ampmut_self(
                                                         self.infcx.tcx,
                                                         local_decl,
@@ -555,7 +542,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                                                     self.infcx.tcx,
                                                     local_decl,
                                                     opt_assignment_rhs_span,
-                                                    *opt_ty_info,
+                                                    opt_ty_info,
                                                 ),
                                             }
                                         };
@@ -564,21 +551,15 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                                 }
                             }
 
-                            LocalInfo::User(ClearCrossCrate::Set(mir::BindingForm::Var(
-                                mir::VarBindingForm {
-                                    binding_mode: ty::BindingMode::BindByReference(_),
-                                    ..
-                                },
-                            ))) => {
+                            LocalInfo::User(mir::BindingForm::Var(mir::VarBindingForm {
+                                binding_mode: ty::BindingMode::BindByReference(_),
+                                ..
+                            })) => {
                                 let pattern_span = local_decl.source_info.span;
                                 suggest_ref_mut(self.infcx.tcx, pattern_span)
                                     .map(|replacement| (true, pattern_span, replacement))
                             }
 
-                            LocalInfo::User(ClearCrossCrate::Clear) => {
-                                bug!("saw cleared local state")
-                            }
-
                             _ => unreachable!(),
                         };
 
@@ -828,7 +809,13 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             let Some(hir::Node::Item(item)) = node else { return; };
             let hir::ItemKind::Fn(.., body_id) = item.kind else { return; };
             let body = self.infcx.tcx.hir().body(body_id);
-            let mut v = V { assign_span: span, err, ty, suggested: false };
+            let mut assign_span = span;
+            // Drop desugaring is done at MIR build so it's not in the HIR
+            if let Some(DesugaringKind::Replace) = span.desugaring_kind() {
+                assign_span.remove_mark();
+            }
+
+            let mut v = V { assign_span, err, ty, suggested: false };
             v.visit_body(body);
             if !v.suggested {
                 err.help(&format!(
@@ -1145,20 +1132,19 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
 pub fn mut_borrow_of_mutable_ref(local_decl: &LocalDecl<'_>, local_name: Option<Symbol>) -> bool {
     debug!("local_info: {:?}, ty.kind(): {:?}", local_decl.local_info, local_decl.ty.kind());
 
-    match local_decl.local_info.as_deref() {
+    match *local_decl.local_info() {
         // Check if mutably borrowing a mutable reference.
-        Some(LocalInfo::User(ClearCrossCrate::Set(mir::BindingForm::Var(
-            mir::VarBindingForm {
-                binding_mode: ty::BindingMode::BindByValue(Mutability::Not), ..
-            },
-        )))) => matches!(local_decl.ty.kind(), ty::Ref(_, _, hir::Mutability::Mut)),
-        Some(LocalInfo::User(ClearCrossCrate::Set(mir::BindingForm::ImplicitSelf(kind)))) => {
+        LocalInfo::User(mir::BindingForm::Var(mir::VarBindingForm {
+            binding_mode: ty::BindingMode::BindByValue(Mutability::Not),
+            ..
+        })) => matches!(local_decl.ty.kind(), ty::Ref(_, _, hir::Mutability::Mut)),
+        LocalInfo::User(mir::BindingForm::ImplicitSelf(kind)) => {
             // Check if the user variable is a `&mut self` and we can therefore
             // suggest removing the `&mut`.
             //
             // Deliberately fall into this case for all implicit self types,
             // so that we don't fall in to the next case with them.
-            *kind == hir::ImplicitSelfKind::MutRef
+            kind == hir::ImplicitSelfKind::MutRef
         }
         _ if Some(kw::SelfLower) == local_name => {
             // Otherwise, check if the name is the `self` keyword - in which case
@@ -1268,7 +1254,7 @@ fn suggest_ampmut<'tcx>(
     (
         suggestability,
         highlight_span,
-        if local_decl.ty.is_region_ptr() {
+        if local_decl.ty.is_ref() {
             format!("&mut {}", ty_mut.ty)
         } else {
             format!("*mut {}", ty_mut.ty)
@@ -1290,7 +1276,7 @@ fn is_closure_or_generator(ty: Ty<'_>) -> bool {
 fn get_mut_span_in_struct_field<'tcx>(
     tcx: TyCtxt<'tcx>,
     ty: Ty<'tcx>,
-    field: mir::Field,
+    field: FieldIdx,
 ) -> Option<Span> {
     // Expect our local to be a reference to a struct of some kind.
     if let ty::Ref(_, ty, _) = ty.kind()
diff --git a/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs b/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
index 1eaf0a2f1..d5ece5743 100644
--- a/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
@@ -1,7 +1,7 @@
 //! Contains utilities for generating suggestions for borrowck errors related to unsatisfied
 //! outlives constraints.
 
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::Diagnostic;
 use rustc_middle::ty::RegionVid;
 use smallvec::SmallVec;
@@ -87,7 +87,7 @@ impl OutlivesSuggestionBuilder {
 
         // Keep track of variables that we have already suggested unifying so that we don't print
         // out silly duplicate messages.
-        let mut unified_already = FxHashSet::default();
+        let mut unified_already = FxIndexSet::default();
 
         for (fr, outlived) in &self.constraints_to_add {
             let Some(fr_name) = self.region_vid_to_name(mbcx, *fr) else {
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
index ffe82b46c..9fcebeb0a 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
@@ -207,7 +207,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             .regioncx
             .placeholders_contained_in(lower_bound)
             .map(|placeholder| {
-                if let Some(id) = placeholder.name.get_id()
+                if let Some(id) = placeholder.bound.kind.get_id()
                     && let Some(placeholder_id) = id.as_local()
                     && let gat_hir_id = hir.local_def_id_to_hir_id(placeholder_id)
                     && let Some(generics_impl) = hir.get_parent(gat_hir_id).generics()
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
index f6881a2e5..f69c4829a 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@@ -6,11 +6,11 @@ use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_middle::ty::print::RegionHighlightMode;
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
-use rustc_middle::ty::{self, DefIdTree, RegionVid, Ty};
+use rustc_middle::ty::{self, RegionVid, Ty};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 
-use crate::{nll::ToRegionVid, universal_regions::DefiningTy, MirBorrowckCtxt};
+use crate::{universal_regions::DefiningTy, MirBorrowckCtxt};
 
 /// A name for a particular region used in emitting diagnostics. This name could be a generated
 /// name like `'1`, a name used by the user like `'a`, or a name like `'static`.
@@ -497,7 +497,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                 //     &
                 //     - let's call the lifetime of this reference `'1`
                 (ty::Ref(region, referent_ty, _), hir::TyKind::Ref(_lifetime, referent_hir_ty)) => {
-                    if region.to_region_vid() == needle_fr {
+                    if region.as_var() == needle_fr {
                         // Just grab the first character, the `&`.
                         let source_map = self.infcx.tcx.sess.source_map();
                         let ampersand_span = source_map.start_point(hir_ty.span);
@@ -598,7 +598,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
         for (kind, hir_arg) in iter::zip(substs, args.args) {
             match (kind.unpack(), hir_arg) {
                 (GenericArgKind::Lifetime(r), hir::GenericArg::Lifetime(lt)) => {
-                    if r.to_region_vid() == needle_fr {
+                    if r.as_var() == needle_fr {
                         return Some(lt);
                     }
                 }
@@ -666,7 +666,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
 
         let return_ty = self.regioncx.universal_regions().unnormalized_output_ty;
         debug!("give_name_if_anonymous_region_appears_in_output: return_ty = {:?}", return_ty);
-        if !tcx.any_free_region_meets(&return_ty, |r| r.to_region_vid() == fr) {
+        if !tcx.any_free_region_meets(&return_ty, |r| r.as_var() == fr) {
             return None;
         }
 
@@ -803,7 +803,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
 
         let tcx = self.infcx.tcx;
 
-        if !tcx.any_free_region_meets(&yield_ty, |r| r.to_region_vid() == fr) {
+        if !tcx.any_free_region_meets(&yield_ty, |r| r.as_var() == fr) {
             return None;
         }
 
diff --git a/compiler/rustc_borrowck/src/diagnostics/var_name.rs b/compiler/rustc_borrowck/src/diagnostics/var_name.rs
index ada3310d8..376415e3d 100644
--- a/compiler/rustc_borrowck/src/diagnostics/var_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/var_name.rs
@@ -1,9 +1,9 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
+use crate::region_infer::RegionInferenceContext;
 use crate::Upvar;
-use crate::{nll::ToRegionVid, region_infer::RegionInferenceContext};
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{Idx, IndexSlice};
 use rustc_middle::mir::{Body, Local};
 use rustc_middle::ty::{RegionVid, TyCtxt};
 use rustc_span::source_map::Span;
@@ -14,7 +14,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         &self,
         tcx: TyCtxt<'tcx>,
         body: &Body<'tcx>,
-        local_names: &IndexVec<Local, Option<Symbol>>,
+        local_names: &IndexSlice<Local, Option<Symbol>>,
         upvars: &[Upvar<'tcx>],
         fr: RegionVid,
     ) -> Option<(Option<Symbol>, Span)> {
@@ -46,7 +46,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             self.universal_regions().defining_ty.upvar_tys().position(|upvar_ty| {
                 debug!("get_upvar_index_for_region: upvar_ty={upvar_ty:?}");
                 tcx.any_free_region_meets(&upvar_ty, |r| {
-                    let r = r.to_region_vid();
+                    let r = r.as_var();
                     debug!("get_upvar_index_for_region: r={r:?} fr={fr:?}");
                     r == fr
                 })
@@ -96,7 +96,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             self.universal_regions().unnormalized_input_tys.iter().skip(implicit_inputs).position(
                 |arg_ty| {
                     debug!("get_argument_index_for_region: arg_ty = {arg_ty:?}");
-                    tcx.any_free_region_meets(arg_ty, |r| r.to_region_vid() == fr)
+                    tcx.any_free_region_meets(arg_ty, |r| r.as_var() == fr)
                 },
             )?;
 
@@ -113,7 +113,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     pub(crate) fn get_argument_name_and_span_for_region(
         &self,
         body: &Body<'tcx>,
-        local_names: &IndexVec<Local, Option<Symbol>>,
+        local_names: &IndexSlice<Local, Option<Symbol>>,
         argument_index: usize,
     ) -> (Option<Symbol>, Span) {
         let implicit_inputs = self.universal_regions().defining_ty.implicit_inputs();
diff --git a/compiler/rustc_borrowck/src/invalidation.rs b/compiler/rustc_borrowck/src/invalidation.rs
index 6217676d5..498d254da 100644
--- a/compiler/rustc_borrowck/src/invalidation.rs
+++ b/compiler/rustc_borrowck/src/invalidation.rs
@@ -79,6 +79,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
             }
             // Only relevant for mir typeck
             StatementKind::AscribeUserType(..)
+            // Only relevant for unsafeck
+            | StatementKind::PlaceMention(..)
             // Doesn't have any language semantics
             | StatementKind::Coverage(..)
             // Does not actually affect borrowck
@@ -118,21 +120,12 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                     LocalMutationIsAllowed::Yes,
                 );
             }
-            TerminatorKind::DropAndReplace {
-                place: drop_place,
-                value: new_value,
-                target: _,
-                unwind: _,
-            } => {
-                self.mutate_place(location, *drop_place, Deep);
-                self.consume_operand(location, new_value);
-            }
             TerminatorKind::Call {
                 func,
                 args,
                 destination,
                 target: _,
-                cleanup: _,
+                unwind: _,
                 from_hir_call: _,
                 fn_span: _,
             } => {
@@ -142,7 +135,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                 }
                 self.mutate_place(location, *destination, Deep);
             }
-            TerminatorKind::Assert { cond, expected: _, msg, target: _, cleanup: _ } => {
+            TerminatorKind::Assert { cond, expected: _, msg, target: _, unwind: _ } => {
                 self.consume_operand(location, cond);
                 use rustc_middle::mir::AssertKind;
                 if let AssertKind::BoundsCheck { len, index } = msg {
@@ -180,7 +173,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                 options: _,
                 line_spans: _,
                 destination: _,
-                cleanup: _,
+                unwind: _,
             } => {
                 for op in operands {
                     match op {
@@ -205,7 +198,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                 }
             }
             TerminatorKind::Goto { target: _ }
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Unreachable
             | TerminatorKind::FalseEdge { real_target: _, imaginary_target: _ }
             | TerminatorKind::FalseUnwind { real_target: _, unwind: _ } => {
diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index 0f591460e..a4b285a34 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -1,12 +1,11 @@
 //! This query borrow-checks the MIR to (further) ensure it is not broken.
 
-#![allow(rustc::potential_query_instability)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(never_type)]
-#![feature(once_cell)]
+#![feature(lazy_cell)]
 #![feature(rustc_attrs)]
 #![feature(stmt_expr_attributes)]
 #![feature(trusted_step)]
@@ -18,14 +17,13 @@ extern crate rustc_middle;
 #[macro_use]
 extern crate tracing;
 
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
-use rustc_data_structures::vec_map::VecMap;
 use rustc_errors::{Diagnostic, DiagnosticBuilder, DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::bit_set::ChunkedBitSet;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_infer::infer::{
     DefiningAnchor, InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin, TyCtxtInferExt,
 };
@@ -35,12 +33,13 @@ use rustc_middle::mir::{
     Place, PlaceElem, PlaceRef, VarDebugInfoContents,
 };
 use rustc_middle::mir::{AggregateKind, BasicBlock, BorrowCheckResult, BorrowKind};
-use rustc_middle::mir::{Field, ProjectionElem, Promoted, Rvalue, Statement, StatementKind};
 use rustc_middle::mir::{InlineAsmOperand, Terminator, TerminatorKind};
+use rustc_middle::mir::{ProjectionElem, Promoted, Rvalue, Statement, StatementKind};
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, CapturedPlace, ParamEnv, RegionVid, TyCtxt};
 use rustc_session::lint::builtin::UNUSED_MUT;
 use rustc_span::{Span, Symbol};
+use rustc_target::abi::FieldIdx;
 
 use either::Either;
 use smallvec::SmallVec;
@@ -95,13 +94,13 @@ pub mod consumers;
 
 use borrow_set::{BorrowData, BorrowSet};
 use dataflow::{BorrowIndex, BorrowckFlowState as Flows, BorrowckResults, Borrows};
-use nll::{PoloniusOutput, ToRegionVid};
+use nll::PoloniusOutput;
 use place_ext::PlaceExt;
 use places_conflict::{places_conflict, PlaceConflictBias};
 use region_infer::RegionInferenceContext;
 use renumber::RegionCtxt;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 // FIXME(eddyb) perhaps move this somewhere more centrally.
 #[derive(Debug)]
@@ -142,7 +141,7 @@ fn mir_borrowck(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> &Bor
         debug!("Skipping borrowck because of injected body");
         // Let's make up a borrowck result! Fun times!
         let result = BorrowCheckResult {
-            concrete_opaque_types: VecMap::new(),
+            concrete_opaque_types: FxIndexMap::default(),
             closure_requirements: None,
             used_mut_upvars: SmallVec::new(),
             tainted_by_errors: None,
@@ -155,7 +154,7 @@ fn mir_borrowck(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> &Bor
     let infcx =
         tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bind(hir_owner.def_id)).build();
     let input_body: &Body<'_> = &input_body.borrow();
-    let promoted: &IndexVec<_, _> = &promoted.borrow();
+    let promoted: &IndexSlice<_, _> = &promoted.borrow();
     let opt_closure_req = do_mir_borrowck(&infcx, input_body, promoted, false).0;
     debug!("mir_borrowck done");
 
@@ -171,7 +170,7 @@ fn mir_borrowck(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> &Bor
 fn do_mir_borrowck<'tcx>(
     infcx: &InferCtxt<'tcx>,
     input_body: &Body<'tcx>,
-    input_promoted: &IndexVec<Promoted, Body<'tcx>>,
+    input_promoted: &IndexSlice<Promoted, Body<'tcx>>,
     return_body_with_facts: bool,
 ) -> (BorrowCheckResult<'tcx>, Option<Box<BodyWithBorrowckFacts<'tcx>>>) {
     let def = input_body.source.with_opt_param().as_local().unwrap();
@@ -224,7 +223,7 @@ fn do_mir_borrowck<'tcx>(
     // be modified (in place) to contain non-lexical lifetimes. It
     // will have a lifetime tied to the inference context.
     let mut body_owned = input_body.clone();
-    let mut promoted = input_promoted.clone();
+    let mut promoted = input_promoted.to_owned();
     let free_regions =
         nll::replace_regions_in_mir(&infcx, param_env, &mut body_owned, &mut promoted);
     let body = &body_owned; // no further changes
@@ -404,7 +403,7 @@ fn do_mir_borrowck<'tcx>(
     // Note that this set is expected to be small - only upvars from closures
     // would have a chance of erroneously adding non-user-defined mutable vars
     // to the set.
-    let temporary_used_locals: FxHashSet<Local> = mbcx
+    let temporary_used_locals: FxIndexSet<Local> = mbcx
         .used_mut
         .iter()
         .filter(|&local| !mbcx.body.local_decls[*local].is_user_variable())
@@ -491,7 +490,7 @@ pub struct BodyWithBorrowckFacts<'tcx> {
 
 pub struct BorrowckInferCtxt<'cx, 'tcx> {
     pub(crate) infcx: &'cx InferCtxt<'tcx>,
-    pub(crate) reg_var_to_origin: RefCell<FxHashMap<ty::RegionVid, RegionCtxt>>,
+    pub(crate) reg_var_to_origin: RefCell<FxIndexMap<ty::RegionVid, RegionCtxt>>,
 }
 
 impl<'cx, 'tcx> BorrowckInferCtxt<'cx, 'tcx> {
@@ -508,20 +507,13 @@ impl<'cx, 'tcx> BorrowckInferCtxt<'cx, 'tcx> {
         F: Fn() -> RegionCtxt,
     {
         let next_region = self.infcx.next_region_var(origin);
-        let vid = next_region
-            .as_var()
-            .unwrap_or_else(|| bug!("expected RegionKind::RegionVar on {:?}", next_region));
+        let vid = next_region.as_var();
 
-        if cfg!(debug_assertions) {
+        if cfg!(debug_assertions) && !self.inside_canonicalization_ctxt() {
             debug!("inserting vid {:?} with origin {:?} into var_to_origin", vid, origin);
             let ctxt = get_ctxt_fn();
             let mut var_to_origin = self.reg_var_to_origin.borrow_mut();
-            let prev = var_to_origin.insert(vid, ctxt);
-
-            // This only makes sense if not called in a canonicalization context. If this
-            // ever changes we either want to get rid of `BorrowckInferContext::reg_var_to_origin`
-            // or modify how we track nll region vars for that map.
-            assert!(matches!(prev, None));
+            var_to_origin.insert(vid, ctxt);
         }
 
         next_region
@@ -537,20 +529,13 @@ impl<'cx, 'tcx> BorrowckInferCtxt<'cx, 'tcx> {
         F: Fn() -> RegionCtxt,
     {
         let next_region = self.infcx.next_nll_region_var(origin.clone());
-        let vid = next_region
-            .as_var()
-            .unwrap_or_else(|| bug!("expected RegionKind::RegionVar on {:?}", next_region));
+        let vid = next_region.as_var();
 
-        if cfg!(debug_assertions) {
+        if cfg!(debug_assertions) && !self.inside_canonicalization_ctxt() {
             debug!("inserting vid {:?} with origin {:?} into var_to_origin", vid, origin);
             let ctxt = get_ctxt_fn();
             let mut var_to_origin = self.reg_var_to_origin.borrow_mut();
-            let prev = var_to_origin.insert(vid, ctxt);
-
-            // This only makes sense if not called in a canonicalization context. If this
-            // ever changes we either want to get rid of `BorrowckInferContext::reg_var_to_origin`
-            // or modify how we track nll region vars for that map.
-            assert!(matches!(prev, None));
+            var_to_origin.insert(vid, ctxt);
         }
 
         next_region
@@ -588,7 +573,7 @@ struct MirBorrowckCtxt<'cx, 'tcx> {
     /// borrow errors that is handled by the `reservation_error_reported` field as the inclusion
     /// of the `Span` type (while required to mute some errors) stops the muting of the reservation
     /// errors.
-    access_place_error_reported: FxHashSet<(Place<'tcx>, Span)>,
+    access_place_error_reported: FxIndexSet<(Place<'tcx>, Span)>,
     /// This field keeps track of when borrow conflict errors are reported
     /// for reservations, so that we don't report seemingly duplicate
     /// errors for corresponding activations.
@@ -596,20 +581,20 @@ struct MirBorrowckCtxt<'cx, 'tcx> {
     // FIXME: ideally this would be a set of `BorrowIndex`, not `Place`s,
     // but it is currently inconvenient to track down the `BorrowIndex`
     // at the time we detect and report a reservation error.
-    reservation_error_reported: FxHashSet<Place<'tcx>>,
+    reservation_error_reported: FxIndexSet<Place<'tcx>>,
     /// This fields keeps track of the `Span`s that we have
     /// used to report extra information for `FnSelfUse`, to avoid
     /// unnecessarily verbose errors.
-    fn_self_span_reported: FxHashSet<Span>,
+    fn_self_span_reported: FxIndexSet<Span>,
     /// This field keeps track of errors reported in the checking of uninitialized variables,
     /// so that we don't report seemingly duplicate errors.
-    uninitialized_error_reported: FxHashSet<PlaceRef<'tcx>>,
+    uninitialized_error_reported: FxIndexSet<PlaceRef<'tcx>>,
     /// This field keeps track of all the local variables that are declared mut and are mutated.
     /// Used for the warning issued by an unused mutable local variable.
-    used_mut: FxHashSet<Local>,
+    used_mut: FxIndexSet<Local>,
     /// If the function we're checking is a closure, then we'll need to report back the list of
     /// mutable upvars that have been used. This field keeps track of them.
-    used_mut_upvars: SmallVec<[Field; 8]>,
+    used_mut_upvars: SmallVec<[FieldIdx; 8]>,
     /// Region inference context. This contains the results from region inference and lets us e.g.
     /// find out which CFG points are contained in each borrow region.
     regioncx: Rc<RegionInferenceContext<'tcx>>,
@@ -628,7 +613,7 @@ struct MirBorrowckCtxt<'cx, 'tcx> {
 
     /// Record the region names generated for each region in the given
     /// MIR def so that we can reuse them later in help/error messages.
-    region_names: RefCell<FxHashMap<RegionVid, RegionName>>,
+    region_names: RefCell<FxIndexMap<RegionVid, RegionName>>,
 
     /// The counter for generating new region names.
     next_region_name: RefCell<usize>,
@@ -691,6 +676,8 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
             }
             // Only relevant for mir typeck
             StatementKind::AscribeUserType(..)
+            // Only relevant for unsafeck
+            | StatementKind::PlaceMention(..)
             // Doesn't have any language semantics
             | StatementKind::Coverage(..)
             // These do not actually affect borrowck
@@ -744,21 +731,12 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
                     flow_state,
                 );
             }
-            TerminatorKind::DropAndReplace {
-                place: drop_place,
-                value: new_value,
-                target: _,
-                unwind: _,
-            } => {
-                self.mutate_place(loc, (*drop_place, span), Deep, flow_state);
-                self.consume_operand(loc, (new_value, span), flow_state);
-            }
             TerminatorKind::Call {
                 func,
                 args,
                 destination,
                 target: _,
-                cleanup: _,
+                unwind: _,
                 from_hir_call: _,
                 fn_span: _,
             } => {
@@ -768,7 +746,7 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
                 }
                 self.mutate_place(loc, (*destination, span), Deep, flow_state);
             }
-            TerminatorKind::Assert { cond, expected: _, msg, target: _, cleanup: _ } => {
+            TerminatorKind::Assert { cond, expected: _, msg, target: _, unwind: _ } => {
                 self.consume_operand(loc, (cond, span), flow_state);
                 use rustc_middle::mir::AssertKind;
                 if let AssertKind::BoundsCheck { len, index } = msg {
@@ -788,7 +766,7 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
                 options: _,
                 line_spans: _,
                 destination: _,
-                cleanup: _,
+                unwind: _,
             } => {
                 for op in operands {
                     match op {
@@ -819,7 +797,7 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
             }
 
             TerminatorKind::Goto { target: _ }
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Unreachable
             | TerminatorKind::Resume
             | TerminatorKind::Return
@@ -863,11 +841,10 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
                 }
             }
 
-            TerminatorKind::Abort
+            TerminatorKind::Terminate
             | TerminatorKind::Assert { .. }
             | TerminatorKind::Call { .. }
             | TerminatorKind::Drop { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::FalseEdge { real_target: _, imaginary_target: _ }
             | TerminatorKind::FalseUnwind { real_target: _, unwind: _ }
             | TerminatorKind::Goto { .. }
@@ -1185,12 +1162,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                             this.buffer_error(err);
                         }
                         WriteKind::StorageDeadOrDrop => this
-                            .report_borrowed_value_does_not_live_long_enough(
-                                location,
-                                borrow,
-                                place_span,
-                                Some(kind),
-                            ),
+                            .report_storage_dead_or_drop_of_borrowed(location, place_span, borrow),
                         WriteKind::Mutate => {
                             this.report_illegal_mutation_of_borrowed(location, place_span, borrow)
                         }
@@ -1367,7 +1339,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                             self.infcx.tcx.mir_borrowck(def_id);
                         debug!("{:?} used_mut_upvars={:?}", def_id, used_mut_upvars);
                         for field in used_mut_upvars {
-                            self.propagate_closure_used_mut_upvar(&operands[field.index()]);
+                            self.propagate_closure_used_mut_upvar(&operands[*field]);
                         }
                     }
                     AggregateKind::Adt(..)
@@ -2302,7 +2274,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
     /// then returns the index of the field being projected. Note that this closure will always
     /// be `self` in the current MIR, because that is the only time we directly access the fields
     /// of a closure type.
-    fn is_upvar_field_projection(&self, place_ref: PlaceRef<'tcx>) -> Option<Field> {
+    fn is_upvar_field_projection(&self, place_ref: PlaceRef<'tcx>) -> Option<FieldIdx> {
         path_utils::is_upvar_field_projection(self.infcx.tcx, &self.upvars, place_ref, self.body())
     }
 
@@ -2334,7 +2306,7 @@ mod error {
         /// same primary span come out in a consistent order.
         buffered_move_errors:
             BTreeMap<Vec<MoveOutIndex>, (PlaceRef<'tcx>, DiagnosticBuilder<'tcx, ErrorGuaranteed>)>,
-        buffered_mut_errors: FxHashMap<Span, (DiagnosticBuilder<'tcx, ErrorGuaranteed>, usize)>,
+        buffered_mut_errors: FxIndexMap<Span, (DiagnosticBuilder<'tcx, ErrorGuaranteed>, usize)>,
         /// Diagnostics to be reported buffer.
         buffered: Vec<Diagnostic>,
         /// Set to Some if we emit an error during borrowck
diff --git a/compiler/rustc_borrowck/src/member_constraints.rs b/compiler/rustc_borrowck/src/member_constraints.rs
index 4af324f74..f637e6a95 100644
--- a/compiler/rustc_borrowck/src/member_constraints.rs
+++ b/compiler/rustc_borrowck/src/member_constraints.rs
@@ -1,8 +1,8 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_data_structures::captures::Captures;
-use rustc_data_structures::fx::FxHashMap;
-use rustc_index::vec::IndexVec;
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::infer::MemberConstraint;
 use rustc_middle::ty::{self, Ty};
 use rustc_span::Span;
@@ -18,7 +18,7 @@ where
 {
     /// Stores the first "member" constraint for a given `R0`. This is an
     /// index into the `constraints` vector below.
-    first_constraints: FxHashMap<R, NllMemberConstraintIndex>,
+    first_constraints: FxIndexMap<R, NllMemberConstraintIndex>,
 
     /// Stores the data about each `R0 member of [R1..Rn]` constraint.
     /// These are organized into a linked list, so each constraint
@@ -132,7 +132,7 @@ where
 
         let MemberConstraintSet { first_constraints, mut constraints, choice_regions } = self;
 
-        let mut first_constraints2 = FxHashMap::default();
+        let mut first_constraints2 = FxIndexMap::default();
         first_constraints2.reserve(first_constraints.len());
 
         for (r1, start1) in first_constraints {
@@ -215,7 +215,7 @@ where
 /// target_list: A -> B -> C -> D -> E -> F -> (None)
 /// ```
 fn append_list(
-    constraints: &mut IndexVec<NllMemberConstraintIndex, NllMemberConstraint<'_>>,
+    constraints: &mut IndexSlice<NllMemberConstraintIndex, NllMemberConstraint<'_>>,
     target_list: NllMemberConstraintIndex,
     source_list: NllMemberConstraintIndex,
 ) {
diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index 96228338a..59a3ab318 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -2,15 +2,15 @@
 #![deny(rustc::diagnostic_outside_of_impl)]
 //! The entry point of the NLL borrow checker.
 
-use rustc_data_structures::vec_map::VecMap;
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::def_id::LocalDefId;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::IndexSlice;
 use rustc_middle::mir::{create_dump_file, dump_enabled, dump_mir, PassWhere};
 use rustc_middle::mir::{
     BasicBlock, Body, ClosureOutlivesSubject, ClosureRegionRequirements, LocalKind, Location,
     Promoted,
 };
-use rustc_middle::ty::{self, OpaqueHiddenType, Region, RegionVid, TyCtxt};
+use rustc_middle::ty::{self, OpaqueHiddenType, TyCtxt};
 use rustc_span::symbol::sym;
 use std::env;
 use std::io;
@@ -44,7 +44,7 @@ pub type PoloniusOutput = Output<RustcFacts>;
 /// closure requirements to propagate, and any generated errors.
 pub(crate) struct NllOutput<'tcx> {
     pub regioncx: RegionInferenceContext<'tcx>,
-    pub opaque_type_values: VecMap<LocalDefId, OpaqueHiddenType<'tcx>>,
+    pub opaque_type_values: FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>,
     pub polonius_input: Option<Box<AllFacts>>,
     pub polonius_output: Option<Rc<PoloniusOutput>>,
     pub opt_closure_req: Option<ClosureRegionRequirements<'tcx>>,
@@ -59,7 +59,7 @@ pub(crate) fn replace_regions_in_mir<'tcx>(
     infcx: &BorrowckInferCtxt<'_, 'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     body: &mut Body<'tcx>,
-    promoted: &mut IndexVec<Promoted, Body<'tcx>>,
+    promoted: &mut IndexSlice<Promoted, Body<'tcx>>,
 ) -> UniversalRegions<'tcx> {
     let def = body.source.with_opt_param().as_local().unwrap();
 
@@ -158,7 +158,7 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
     infcx: &BorrowckInferCtxt<'_, 'tcx>,
     universal_regions: UniversalRegions<'tcx>,
     body: &Body<'tcx>,
-    promoted: &IndexVec<Promoted, Body<'tcx>>,
+    promoted: &IndexSlice<Promoted, Body<'tcx>>,
     location_table: &LocationTable,
     param_env: ty::ParamEnv<'tcx>,
     flow_inits: &mut ResultsCursor<'cx, 'tcx, MaybeInitializedPlaces<'cx, 'tcx>>,
@@ -377,7 +377,7 @@ pub(super) fn dump_annotation<'tcx>(
     body: &Body<'tcx>,
     regioncx: &RegionInferenceContext<'tcx>,
     closure_region_requirements: &Option<ClosureRegionRequirements<'tcx>>,
-    opaque_type_values: &VecMap<LocalDefId, OpaqueHiddenType<'tcx>>,
+    opaque_type_values: &FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>,
     errors: &mut crate::error::BorrowckErrors<'tcx>,
 ) {
     let tcx = infcx.tcx;
@@ -444,27 +444,6 @@ fn for_each_region_constraint<'tcx>(
     Ok(())
 }
 
-/// Right now, we piggy back on the `ReVar` to store our NLL inference
-/// regions. These are indexed with `RegionVid`. This method will
-/// assert that the region is a `ReVar` and extract its internal index.
-/// This is reasonable because in our MIR we replace all universal regions
-/// with inference variables.
-pub trait ToRegionVid {
-    fn to_region_vid(self) -> RegionVid;
-}
-
-impl<'tcx> ToRegionVid for Region<'tcx> {
-    fn to_region_vid(self) -> RegionVid {
-        if let ty::ReVar(vid) = *self { vid } else { bug!("region is not an ReVar: {:?}", self) }
-    }
-}
-
-impl ToRegionVid for RegionVid {
-    fn to_region_vid(self) -> RegionVid {
-        self
-    }
-}
-
 pub(crate) trait ConstraintDescription {
     fn description(&self) -> &'static str;
 }
diff --git a/compiler/rustc_borrowck/src/path_utils.rs b/compiler/rustc_borrowck/src/path_utils.rs
index f8a99a269..ea9f8683c 100644
--- a/compiler/rustc_borrowck/src/path_utils.rs
+++ b/compiler/rustc_borrowck/src/path_utils.rs
@@ -7,8 +7,9 @@ use crate::BorrowIndex;
 use crate::Upvar;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_middle::mir::BorrowKind;
-use rustc_middle::mir::{BasicBlock, Body, Field, Location, Place, PlaceRef, ProjectionElem};
+use rustc_middle::mir::{BasicBlock, Body, Location, Place, PlaceRef, ProjectionElem};
 use rustc_middle::ty::TyCtxt;
+use rustc_target::abi::FieldIdx;
 
 /// Returns `true` if the borrow represented by `kind` is
 /// allowed to be split into separate Reservation and
@@ -148,7 +149,7 @@ pub(crate) fn is_upvar_field_projection<'tcx>(
     upvars: &[Upvar<'tcx>],
     place_ref: PlaceRef<'tcx>,
     body: &Body<'tcx>,
-) -> Option<Field> {
+) -> Option<FieldIdx> {
     let mut place_ref = place_ref;
     let mut by_ref = false;
 
diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs
index e6195de40..729f3dbff 100644
--- a/compiler/rustc_borrowck/src/region_infer/mod.rs
+++ b/compiler/rustc_borrowck/src/region_infer/mod.rs
@@ -3,11 +3,11 @@ use std::rc::Rc;
 
 use rustc_data_structures::binary_search_util;
 use rustc_data_structures::frozen::Frozen;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::scc::Sccs;
 use rustc_errors::Diagnostic;
 use rustc_hir::def_id::CRATE_DEF_ID;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_infer::infer::outlives::test_type_match;
 use rustc_infer::infer::region_constraints::{GenericKind, VarInfos, VerifyBound, VerifyIfEq};
 use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin};
@@ -27,7 +27,7 @@ use crate::{
     },
     diagnostics::{RegionErrorKind, RegionErrors, UniverseInfo},
     member_constraints::{MemberConstraintSet, NllMemberConstraintIndex},
-    nll::{PoloniusOutput, ToRegionVid},
+    nll::PoloniusOutput,
     region_infer::reverse_sccs::ReverseSccGraph,
     region_infer::values::{
         LivenessValues, PlaceholderIndices, RegionElement, RegionValueElements, RegionValues,
@@ -88,7 +88,7 @@ pub struct RegionInferenceContext<'tcx> {
     member_constraints_applied: Vec<AppliedMemberConstraint>,
 
     /// Map universe indexes to information on why we created it.
-    universe_causes: FxHashMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
+    universe_causes: FxIndexMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
 
     /// Contains the minimum universe of any variable within the same
     /// SCC. We will ensure that no SCC contains values that are not
@@ -255,15 +255,16 @@ fn sccs_info<'cx, 'tcx>(
     let var_to_origin = infcx.reg_var_to_origin.borrow();
 
     let mut var_to_origin_sorted = var_to_origin.clone().into_iter().collect::<Vec<_>>();
-    var_to_origin_sorted.sort_by(|a, b| a.0.cmp(&b.0));
-    let mut debug_str = "region variables to origins:\n".to_string();
+    var_to_origin_sorted.sort_by_key(|vto| vto.0);
+
+    let mut reg_vars_to_origins_str = "region variables to origins:\n".to_string();
     for (reg_var, origin) in var_to_origin_sorted.into_iter() {
-        debug_str.push_str(&format!("{:?}: {:?}\n", reg_var, origin));
+        reg_vars_to_origins_str.push_str(&format!("{:?}: {:?}\n", reg_var, origin));
     }
-    debug!(debug_str);
+    debug!("{}", reg_vars_to_origins_str);
 
     let num_components = sccs.scc_data().ranges().len();
-    let mut components = vec![FxHashSet::default(); num_components];
+    let mut components = vec![FxIndexSet::default(); num_components];
 
     for (reg_var_idx, scc_idx) in sccs.scc_indices().iter().enumerate() {
         let reg_var = ty::RegionVid::from_usize(reg_var_idx);
@@ -275,12 +276,12 @@ fn sccs_info<'cx, 'tcx>(
     for (scc_idx, reg_vars_origins) in components.iter().enumerate() {
         let regions_info = reg_vars_origins.clone().into_iter().collect::<Vec<_>>();
         components_str.push_str(&format!(
-            "{:?}: {:?})",
+            "{:?}: {:?},\n)",
             ConstraintSccIndex::from_usize(scc_idx),
             regions_info,
         ))
     }
-    debug!(components_str);
+    debug!("{}", components_str);
 
     // calculate the best representative for each component
     let components_representatives = components
@@ -295,9 +296,9 @@ fn sccs_info<'cx, 'tcx>(
 
             (ConstraintSccIndex::from_usize(scc_idx), repr)
         })
-        .collect::<FxHashMap<_, _>>();
+        .collect::<FxIndexMap<_, _>>();
 
-    let mut scc_node_to_edges = FxHashMap::default();
+    let mut scc_node_to_edges = FxIndexMap::default();
     for (scc_idx, repr) in components_representatives.iter() {
         let edges_range = sccs.scc_data().ranges()[*scc_idx].clone();
         let edges = &sccs.scc_data().all_successors()[edges_range];
@@ -325,7 +326,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
         outlives_constraints: OutlivesConstraintSet<'tcx>,
         member_constraints_in: MemberConstraintSet<'tcx, RegionVid>,
-        universe_causes: FxHashMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
+        universe_causes: FxIndexMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
         type_tests: Vec<TypeTest<'tcx>>,
         liveness_constraints: LivenessValues<RegionVid>,
         elements: &Rc<RegionValueElements>,
@@ -398,7 +399,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// the minimum, or narrowest, universe.
     fn compute_scc_universes(
         constraint_sccs: &Sccs<RegionVid, ConstraintSccIndex>,
-        definitions: &IndexVec<RegionVid, RegionDefinition<'tcx>>,
+        definitions: &IndexSlice<RegionVid, RegionDefinition<'tcx>>,
     ) -> IndexVec<ConstraintSccIndex, ty::UniverseIndex> {
         let num_sccs = constraint_sccs.num_sccs();
         let mut scc_universes = IndexVec::from_elem_n(ty::UniverseIndex::MAX, num_sccs);
@@ -485,7 +486,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// more details.
     fn compute_scc_representatives(
         constraints_scc: &Sccs<RegionVid, ConstraintSccIndex>,
-        definitions: &IndexVec<RegionVid, RegionDefinition<'tcx>>,
+        definitions: &IndexSlice<RegionVid, RegionDefinition<'tcx>>,
     ) -> IndexVec<ConstraintSccIndex, ty::RegionVid> {
         let num_sccs = constraints_scc.num_sccs();
         let next_region_vid = definitions.next_index();
@@ -522,6 +523,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// outlives `'a` and hence contains R0 and R1.
     fn init_free_and_bound_regions(&mut self) {
         // Update the names (if any)
+        // This iterator has unstable order but we collect it all into an IndexVec
         for (external_name, variable) in self.universal_regions.named_universal_regions() {
             debug!(
                 "init_universal_regions: region {:?} has external name {:?}",
@@ -591,14 +593,14 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// Returns `true` if the region `r` contains the point `p`.
     ///
     /// Panics if called before `solve()` executes,
-    pub(crate) fn region_contains(&self, r: impl ToRegionVid, p: impl ToElementIndex) -> bool {
-        let scc = self.constraint_sccs.scc(r.to_region_vid());
+    pub(crate) fn region_contains(&self, r: RegionVid, p: impl ToElementIndex) -> bool {
+        let scc = self.constraint_sccs.scc(r);
         self.scc_values.contains(scc, p)
     }
 
     /// Returns access to the value of `r` for debugging purposes.
     pub(crate) fn region_value_str(&self, r: RegionVid) -> String {
-        let scc = self.constraint_sccs.scc(r.to_region_vid());
+        let scc = self.constraint_sccs.scc(r);
         self.scc_values.region_value_str(scc)
     }
 
@@ -606,24 +608,21 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         &'a self,
         r: RegionVid,
     ) -> impl Iterator<Item = ty::PlaceholderRegion> + 'a {
-        let scc = self.constraint_sccs.scc(r.to_region_vid());
+        let scc = self.constraint_sccs.scc(r);
         self.scc_values.placeholders_contained_in(scc)
     }
 
     /// Returns access to the value of `r` for debugging purposes.
     pub(crate) fn region_universe(&self, r: RegionVid) -> ty::UniverseIndex {
-        let scc = self.constraint_sccs.scc(r.to_region_vid());
+        let scc = self.constraint_sccs.scc(r);
         self.scc_universes[scc]
     }
 
     /// Once region solving has completed, this function will return
     /// the member constraints that were applied to the value of a given
     /// region `r`. See `AppliedMemberConstraint`.
-    pub(crate) fn applied_member_constraints(
-        &self,
-        r: impl ToRegionVid,
-    ) -> &[AppliedMemberConstraint] {
-        let scc = self.constraint_sccs.scc(r.to_region_vid());
+    pub(crate) fn applied_member_constraints(&self, r: RegionVid) -> &[AppliedMemberConstraint] {
+        let scc = self.constraint_sccs.scc(r);
         binary_search_util::binary_search_slice(
             &self.member_constraints_applied,
             |applied| applied.member_region_scc,
@@ -918,7 +917,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         // Sometimes we register equivalent type-tests that would
         // result in basically the exact same error being reported to
         // the user. Avoid that.
-        let mut deduplicate_errors = FxHashSet::default();
+        let mut deduplicate_errors = FxIndexSet::default();
 
         for type_test in &self.type_tests {
             debug!("check_type_test: {:?}", type_test);
@@ -1131,7 +1130,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             let r_vid = self.to_region_vid(r);
             let r_scc = self.constraint_sccs.scc(r_vid);
 
-            // The challenge if this. We have some region variable `r`
+            // The challenge is this. We have some region variable `r`
             // whose value is a set of CFG points and universal
             // regions. We want to find if that set is *equivalent* to
             // any of the named regions found in the closure.
@@ -1504,6 +1503,8 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         // the outlives suggestions or the debug output from `#[rustc_regions]` would be
         // duplicated. The polonius subset errors are deduplicated here, while keeping the
         // CFG-location ordering.
+        // We can iterate the HashMap here because the result is sorted afterwards.
+        #[allow(rustc::potential_query_instability)]
         let mut subset_errors: Vec<_> = polonius_output
             .subset_errors
             .iter()
@@ -2213,7 +2214,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         // is in the same SCC or something. In that case, find what
         // appears to be the most interesting point to report to the
         // user via an even more ad-hoc guess.
-        categorized_path.sort_by(|p0, p1| p0.category.cmp(&p1.category));
+        categorized_path.sort_by_key(|p| p.category);
         debug!("sorted_path={:#?}", categorized_path);
 
         (categorized_path.remove(0), extra_info)
@@ -2230,7 +2231,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         r: RegionVid,
         body: &Body<'_>,
     ) -> Option<Location> {
-        let scc = self.constraint_sccs.scc(r.to_region_vid());
+        let scc = self.constraint_sccs.scc(r);
         let locations = self.scc_values.locations_outlived_by(scc);
         for location in locations {
             let bb = &body[location.block];
diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
index c550e37c6..2b16655cf 100644
--- a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
+++ b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
@@ -1,5 +1,4 @@
-use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
-use rustc_data_structures::vec_map::VecMap;
+use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::OpaqueTyOrigin;
@@ -61,11 +60,11 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     pub(crate) fn infer_opaque_types(
         &self,
         infcx: &InferCtxt<'tcx>,
-        opaque_ty_decls: VecMap<OpaqueTypeKey<'tcx>, (OpaqueHiddenType<'tcx>, OpaqueTyOrigin)>,
-    ) -> VecMap<LocalDefId, OpaqueHiddenType<'tcx>> {
-        let mut result: VecMap<LocalDefId, OpaqueHiddenType<'tcx>> = VecMap::new();
+        opaque_ty_decls: FxIndexMap<OpaqueTypeKey<'tcx>, (OpaqueHiddenType<'tcx>, OpaqueTyOrigin)>,
+    ) -> FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>> {
+        let mut result: FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>> = FxIndexMap::default();
 
-        let member_constraints: FxHashMap<_, _> = self
+        let member_constraints: FxIndexMap<_, _> = self
             .member_constraints
             .all_indices()
             .map(|ci| (self.member_constraints[ci].key, ci))
@@ -284,7 +283,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         // hidden type is well formed even without those bounds.
         let predicate = ty::Binder::dummy(ty::PredicateKind::WellFormed(definition_ty.into()));
 
-        let id_substs = InternalSubsts::identity_for_item(self.tcx, def_id.to_def_id());
+        let id_substs = InternalSubsts::identity_for_item(self.tcx, def_id);
 
         // Require that the hidden type actually fulfills all the bounds of the opaque type, even without
         // the bounds that the function supplies.
@@ -325,7 +324,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         if errors.is_empty() {
             definition_ty
         } else {
-            let reported = infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+            let reported = infcx.err_ctxt().report_fulfillment_errors(&errors);
             self.tcx.ty_error(reported)
         }
     }
@@ -364,7 +363,7 @@ fn check_opaque_type_parameter_valid(
         OpaqueTyOrigin::TyAlias => {}
     }
     let opaque_generics = tcx.generics_of(opaque_type_key.def_id);
-    let mut seen_params: FxHashMap<_, Vec<_>> = FxHashMap::default();
+    let mut seen_params: FxIndexMap<_, Vec<_>> = FxIndexMap::default();
     for (i, arg) in opaque_type_key.substs.iter().enumerate() {
         let arg_is_param = match arg.unpack() {
             GenericArgKind::Type(ty) => matches!(ty.kind(), ty::Param(_)),
diff --git a/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs b/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs
index 167f66460..23a59c128 100644
--- a/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs
+++ b/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs
@@ -3,7 +3,7 @@
 use crate::constraints::ConstraintSccIndex;
 use crate::RegionInferenceContext;
 use itertools::Itertools;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::vec_graph::VecGraph;
 use rustc_data_structures::graph::WithSuccessors;
 use rustc_middle::ty::RegionVid;
@@ -14,7 +14,7 @@ pub(crate) struct ReverseSccGraph {
     graph: VecGraph<ConstraintSccIndex>,
     /// For each SCC, the range of `universal_regions` that use that SCC as
     /// their value.
-    scc_regions: FxHashMap<ConstraintSccIndex, Range<usize>>,
+    scc_regions: FxIndexMap<ConstraintSccIndex, Range<usize>>,
     /// All of the universal regions, in grouped so that `scc_regions` can
     /// index into here.
     universal_regions: Vec<RegionVid>,
@@ -26,7 +26,7 @@ impl ReverseSccGraph {
         &'a self,
         scc0: ConstraintSccIndex,
     ) -> impl Iterator<Item = RegionVid> + 'a {
-        let mut duplicates = FxHashSet::default();
+        let mut duplicates = FxIndexSet::default();
         self.graph
             .depth_first_search(scc0)
             .flat_map(move |scc1| {
@@ -55,7 +55,7 @@ impl RegionInferenceContext<'_> {
         paired_scc_regions.sort();
         let universal_regions = paired_scc_regions.iter().map(|&(_, region)| region).collect();
 
-        let mut scc_regions = FxHashMap::default();
+        let mut scc_regions = FxIndexMap::default();
         let mut start = 0;
         for (scc, group) in &paired_scc_regions.into_iter().group_by(|(scc, _)| *scc) {
             let group_size = group.count();
diff --git a/compiler/rustc_borrowck/src/renumber.rs b/compiler/rustc_borrowck/src/renumber.rs
index 016f6f78d..94ce29dfe 100644
--- a/compiler/rustc_borrowck/src/renumber.rs
+++ b/compiler/rustc_borrowck/src/renumber.rs
@@ -1,7 +1,7 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 use crate::BorrowckInferCtxt;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::IndexSlice;
 use rustc_infer::infer::NllRegionVariableOrigin;
 use rustc_middle::mir::visit::{MutVisitor, TyContext};
 use rustc_middle::mir::Constant;
@@ -16,35 +16,17 @@ use rustc_span::{Span, Symbol};
 pub fn renumber_mir<'tcx>(
     infcx: &BorrowckInferCtxt<'_, 'tcx>,
     body: &mut Body<'tcx>,
-    promoted: &mut IndexVec<Promoted, Body<'tcx>>,
+    promoted: &mut IndexSlice<Promoted, Body<'tcx>>,
 ) {
     debug!(?body.arg_count);
 
-    let mut visitor = NllVisitor { infcx };
+    let mut renumberer = RegionRenumberer { infcx };
 
     for body in promoted.iter_mut() {
-        visitor.visit_body(body);
+        renumberer.visit_body(body);
     }
 
-    visitor.visit_body(body);
-}
-
-/// Replaces all regions appearing in `value` with fresh inference
-/// variables.
-#[instrument(skip(infcx, get_ctxt_fn), level = "debug")]
-pub(crate) fn renumber_regions<'tcx, T, F>(
-    infcx: &BorrowckInferCtxt<'_, 'tcx>,
-    value: T,
-    get_ctxt_fn: F,
-) -> T
-where
-    T: TypeFoldable<TyCtxt<'tcx>>,
-    F: Fn() -> RegionCtxt,
-{
-    infcx.tcx.fold_regions(value, |_region, _depth| {
-        let origin = NllRegionVariableOrigin::Existential { from_forall: false };
-        infcx.next_nll_region_var(origin, || get_ctxt_fn())
-    })
+    renumberer.visit_body(body);
 }
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
@@ -69,12 +51,10 @@ impl RegionCtxt {
     /// Used to determine the representative of a component in the strongly connected
     /// constraint graph
     pub(crate) fn preference_value(self) -> usize {
-        let _anon = Symbol::intern("anon");
-
         match self {
             RegionCtxt::Unknown => 1,
             RegionCtxt::Existential(None) => 2,
-            RegionCtxt::Existential(Some(_anon)) | RegionCtxt::Free(_anon) => 2,
+            RegionCtxt::Existential(Some(_)) | RegionCtxt::Free(_) => 2,
             RegionCtxt::Location(_) => 3,
             RegionCtxt::TyContext(_) => 4,
             _ => 5,
@@ -82,21 +62,26 @@ impl RegionCtxt {
     }
 }
 
-struct NllVisitor<'a, 'tcx> {
+struct RegionRenumberer<'a, 'tcx> {
     infcx: &'a BorrowckInferCtxt<'a, 'tcx>,
 }
 
-impl<'a, 'tcx> NllVisitor<'a, 'tcx> {
+impl<'a, 'tcx> RegionRenumberer<'a, 'tcx> {
+    /// Replaces all regions appearing in `value` with fresh inference
+    /// variables.
     fn renumber_regions<T, F>(&mut self, value: T, region_ctxt_fn: F) -> T
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
         F: Fn() -> RegionCtxt,
     {
-        renumber_regions(self.infcx, value, region_ctxt_fn)
+        let origin = NllRegionVariableOrigin::Existential { from_forall: false };
+        self.infcx.tcx.fold_regions(value, |_region, _depth| {
+            self.infcx.next_nll_region_var(origin, || region_ctxt_fn())
+        })
     }
 }
 
-impl<'a, 'tcx> MutVisitor<'tcx> for NllVisitor<'a, 'tcx> {
+impl<'a, 'tcx> MutVisitor<'tcx> for RegionRenumberer<'a, 'tcx> {
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
@@ -124,9 +109,9 @@ impl<'a, 'tcx> MutVisitor<'tcx> for NllVisitor<'a, 'tcx> {
     }
 
     #[instrument(skip(self), level = "debug")]
-    fn visit_constant(&mut self, constant: &mut Constant<'tcx>, _location: Location) {
+    fn visit_constant(&mut self, constant: &mut Constant<'tcx>, location: Location) {
         let literal = constant.literal;
-        constant.literal = self.renumber_regions(literal, || RegionCtxt::Location(_location));
+        constant.literal = self.renumber_regions(literal, || RegionCtxt::Location(location));
         debug!("constant: {:#?}", constant);
     }
 }
diff --git a/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs b/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs
index a93561350..71eae7b27 100644
--- a/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs
+++ b/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs
@@ -12,7 +12,6 @@ use rustc_span::{Span, DUMMY_SP};
 
 use crate::{
     constraints::OutlivesConstraint,
-    nll::ToRegionVid,
     region_infer::TypeTest,
     type_check::{Locations, MirTypeckRegionConstraints},
     universal_regions::UniversalRegions,
@@ -198,7 +197,7 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
 
     fn to_region_vid(&mut self, r: ty::Region<'tcx>) -> ty::RegionVid {
         if let ty::RePlaceholder(placeholder) = *r {
-            self.constraints.placeholder_region(self.infcx, placeholder).to_region_vid()
+            self.constraints.placeholder_region(self.infcx, placeholder).as_var()
         } else {
             self.universal_regions.to_region_vid(r)
         }
diff --git a/compiler/rustc_borrowck/src/type_check/input_output.rs b/compiler/rustc_borrowck/src/type_check/input_output.rs
index 717020ea5..17e702eb8 100644
--- a/compiler/rustc_borrowck/src/type_check/input_output.rs
+++ b/compiler/rustc_borrowck/src/type_check/input_output.rs
@@ -19,7 +19,7 @@ use super::{Locations, TypeChecker};
 
 impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     /// Check explicit closure signature annotation,
-    /// e.g., `|x: FxHashMap<_, &'static u32>| ...`.
+    /// e.g., `|x: FxIndexMap<_, &'static u32>| ...`.
     #[instrument(skip(self, body), level = "debug")]
     pub(super) fn check_signature_annotation(&mut self, body: &Body<'tcx>) {
         let mir_def_id = body.source.def_id().expect_local();
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs b/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
index 8023ef60d..2c387edfe 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
@@ -63,7 +63,7 @@ impl LocalUseMap {
         elements: &RegionValueElements,
         body: &Body<'_>,
     ) -> Self {
-        let nones = IndexVec::from_elem_n(None, body.local_decls.len());
+        let nones = IndexVec::from_elem(None, &body.local_decls);
         let mut local_use_map = LocalUseMap {
             first_def_at: nones.clone(),
             first_use_at: nones.clone(),
@@ -76,7 +76,7 @@ impl LocalUseMap {
         }
 
         let mut locals_with_use_data: IndexVec<Local, bool> =
-            IndexVec::from_elem_n(false, body.local_decls.len());
+            IndexVec::from_elem(false, &body.local_decls);
         live_locals.iter().for_each(|&local| locals_with_use_data[local] = true);
 
         LocalUseMapBuild { local_use_map: &mut local_use_map, elements, locals_with_use_data }
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
index a411aec51..f1ad0ca55 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
@@ -11,7 +11,6 @@ use crate::{
     constraints::OutlivesConstraintSet,
     facts::{AllFacts, AllFactsExt},
     location::LocationTable,
-    nll::ToRegionVid,
     region_infer::values::RegionValueElements,
     universal_regions::UniversalRegions,
 };
@@ -80,9 +79,7 @@ fn compute_relevant_live_locals<'tcx>(
 ) -> (Vec<Local>, Vec<Local>) {
     let (boring_locals, relevant_live_locals): (Vec<_>, Vec<_>) =
         body.local_decls.iter_enumerated().partition_map(|(local, local_decl)| {
-            if tcx.all_free_regions_meet(&local_decl.ty, |r| {
-                free_regions.contains(&r.to_region_vid())
-            }) {
+            if tcx.all_free_regions_meet(&local_decl.ty, |r| free_regions.contains(&r.as_var())) {
                 Either::Left(local)
             } else {
                 Either::Right(local)
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index 473c05963..9731b10aa 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -1,4 +1,4 @@
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_index::bit_set::HybridBitSet;
 use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
@@ -56,7 +56,7 @@ pub(super) fn trace<'mir, 'tcx>(
         elements,
         local_use_map,
         move_data,
-        drop_data: FxHashMap::default(),
+        drop_data: FxIndexMap::default(),
     };
 
     let mut results = LivenessResults::new(cx);
@@ -85,7 +85,7 @@ struct LivenessContext<'me, 'typeck, 'flow, 'tcx> {
     move_data: &'me MoveData<'tcx>,
 
     /// Cache for the results of `dropck_outlives` query.
-    drop_data: FxHashMap<Ty<'tcx>, DropData<'tcx>>,
+    drop_data: FxIndexMap<Ty<'tcx>, DropData<'tcx>>,
 
     /// Results of dataflow tracking which variables (and paths) have been
     /// initialized.
@@ -185,7 +185,7 @@ impl<'me, 'typeck, 'flow, 'tcx> LivenessResults<'me, 'typeck, 'flow, 'tcx> {
     fn add_extra_drop_facts(
         &mut self,
         drop_used: Vec<(Local, Location)>,
-        relevant_live_locals: FxHashSet<Local>,
+        relevant_live_locals: FxIndexSet<Local>,
     ) {
         let locations = IntervalSet::new(self.cx.elements.num_points());
 
@@ -435,8 +435,7 @@ impl<'me, 'typeck, 'flow, 'tcx> LivenessResults<'me, 'typeck, 'flow, 'tcx> {
         //
         // What we *actually* generate is a store to a temporary
         // for the call (`TMP = call()...`) and then a
-        // `DropAndReplace` to swap that with `X`
-        // (`DropAndReplace` has very particular semantics).
+        // `Drop(X)` followed by `X = TMP`  to swap that with `X`.
     }
 }
 
diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index a49da3da6..2f10e30be 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -9,13 +9,12 @@ use either::Either;
 
 use hir::OpaqueTyOrigin;
 use rustc_data_structures::frozen::Frozen;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_data_structures::vec_map::VecMap;
+use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::lang_items::LangItem;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_infer::infer::outlives::env::RegionBoundPairs;
 use rustc_infer::infer::region_constraints::RegionConstraintData;
@@ -36,8 +35,9 @@ use rustc_middle::ty::{
     OpaqueHiddenType, OpaqueTypeKey, RegionVid, Ty, TyCtxt, UserType, UserTypeAnnotationIndex,
 };
 use rustc_span::def_id::CRATE_DEF_ID;
+use rustc_span::symbol::sym;
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, FIRST_VARIANT};
 use rustc_trait_selection::traits::query::type_op::custom::scrape_region_constraints;
 use rustc_trait_selection::traits::query::type_op::custom::CustomTypeOp;
 use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput};
@@ -56,7 +56,6 @@ use crate::{
     facts::AllFacts,
     location::LocationTable,
     member_constraints::MemberConstraintSet,
-    nll::ToRegionVid,
     path_utils,
     region_infer::values::{
         LivenessValues, PlaceholderIndex, PlaceholderIndices, RegionValueElements,
@@ -126,7 +125,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
     infcx: &BorrowckInferCtxt<'_, 'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     body: &Body<'tcx>,
-    promoted: &IndexVec<Promoted, Body<'tcx>>,
+    promoted: &IndexSlice<Promoted, Body<'tcx>>,
     universal_regions: &Rc<UniversalRegions<'tcx>>,
     location_table: &LocationTable,
     borrow_set: &BorrowSet<'tcx>,
@@ -145,7 +144,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
         outlives_constraints: OutlivesConstraintSet::default(),
         member_constraints: MemberConstraintSet::default(),
         type_tests: Vec::default(),
-        universe_causes: FxHashMap::default(),
+        universe_causes: FxIndexMap::default(),
     };
 
     let CreateResult {
@@ -293,7 +292,7 @@ enum FieldAccessError {
 /// is a problem.
 struct TypeVerifier<'a, 'b, 'tcx> {
     cx: &'a mut TypeChecker<'b, 'tcx>,
-    promoted: &'b IndexVec<Promoted, Body<'tcx>>,
+    promoted: &'b IndexSlice<Promoted, Body<'tcx>>,
     last_span: Span,
     errors_reported: bool,
 }
@@ -494,7 +493,7 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
 impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
     fn new(
         cx: &'a mut TypeChecker<'b, 'tcx>,
-        promoted: &'b IndexVec<Promoted, Body<'tcx>>,
+        promoted: &'b IndexSlice<Promoted, Body<'tcx>>,
     ) -> Self {
         TypeVerifier { promoted, last_span: cx.body.span, cx, errors_reported: false }
     }
@@ -772,11 +771,14 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
 
         match context {
             PlaceContext::MutatingUse(_) => ty::Invariant,
-            PlaceContext::NonUse(StorageDead | StorageLive | VarDebugInfo) => ty::Invariant,
+            PlaceContext::NonUse(StorageDead | StorageLive | PlaceMention | VarDebugInfo) => {
+                ty::Invariant
+            }
             PlaceContext::NonMutatingUse(
-                Inspect | Copy | Move | SharedBorrow | ShallowBorrow | UniqueBorrow | AddressOf
-                | Projection,
-            ) => ty::Covariant,
+                Inspect | Copy | Move | SharedBorrow | ShallowBorrow | AddressOf | Projection
+            ) => {
+                ty::Covariant
+            },
             PlaceContext::NonUse(AscribeUserTy) => ty::Covariant,
         }
     }
@@ -785,7 +787,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
         &mut self,
         parent: &dyn fmt::Debug,
         base_ty: PlaceTy<'tcx>,
-        field: Field,
+        field: FieldIdx,
         location: Location,
     ) -> Result<Ty<'tcx>, FieldAccessError> {
         let tcx = self.tcx();
@@ -811,7 +813,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
             },
             PlaceTy { ty, variant_index: None } => match *ty.kind() {
                 ty::Adt(adt_def, substs) if !adt_def.is_enum() => {
-                    (adt_def.variant(VariantIdx::new(0)), substs)
+                    (adt_def.variant(FIRST_VARIANT), substs)
                 }
                 ty::Closure(_, substs) => {
                     return match substs
@@ -853,7 +855,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
             },
         };
 
-        if let Some(field) = variant.fields.get(field.index()) {
+        if let Some(field) = variant.fields.get(field) {
             Ok(self.cx.normalize(field.ty(tcx, substs), location))
         } else {
             Err(FieldAccessError::OutOfRange { field_count: variant.fields.len() })
@@ -875,7 +877,7 @@ struct TypeChecker<'a, 'tcx> {
     user_type_annotations: &'a CanonicalUserTypeAnnotations<'tcx>,
     region_bound_pairs: &'a RegionBoundPairs<'tcx>,
     implicit_region_bound: ty::Region<'tcx>,
-    reported_errors: FxHashSet<(Ty<'tcx>, Span)>,
+    reported_errors: FxIndexSet<(Ty<'tcx>, Span)>,
     borrowck_context: &'a mut BorrowCheckContext<'a, 'tcx>,
 }
 
@@ -892,7 +894,7 @@ pub(crate) struct MirTypeckResults<'tcx> {
     pub(crate) constraints: MirTypeckRegionConstraints<'tcx>,
     pub(crate) universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
     pub(crate) opaque_type_values:
-        VecMap<OpaqueTypeKey<'tcx>, (OpaqueHiddenType<'tcx>, OpaqueTyOrigin)>,
+        FxIndexMap<OpaqueTypeKey<'tcx>, (OpaqueHiddenType<'tcx>, OpaqueTyOrigin)>,
 }
 
 /// A collection of region constraints that must be satisfied for the
@@ -925,7 +927,7 @@ pub(crate) struct MirTypeckRegionConstraints<'tcx> {
 
     pub(crate) member_constraints: MemberConstraintSet<'tcx, RegionVid>,
 
-    pub(crate) universe_causes: FxHashMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
+    pub(crate) universe_causes: FxIndexMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
 
     pub(crate) type_tests: Vec<TypeTest<'tcx>>,
 }
@@ -1178,10 +1180,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                         }
                     }
                     Some(l)
-                        if matches!(
-                            body.local_decls[l].local_info,
-                            Some(box LocalInfo::AggregateTemp)
-                        ) =>
+                        if matches!(body.local_decls[l].local_info(), LocalInfo::AggregateTemp) =>
                     {
                         ConstraintCategory::Usage
                     }
@@ -1282,6 +1281,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             | StatementKind::Retag { .. }
             | StatementKind::Coverage(..)
             | StatementKind::ConstEvalCounter
+            | StatementKind::PlaceMention(..)
             | StatementKind::Nop => {}
             StatementKind::Deinit(..) | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
@@ -1301,7 +1301,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         match &term.kind {
             TerminatorKind::Goto { .. }
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::Unreachable
@@ -1312,24 +1312,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 // no checks needed for these
             }
 
-            TerminatorKind::DropAndReplace { place, value, target: _, unwind: _ } => {
-                let place_ty = place.ty(body, tcx).ty;
-                let rv_ty = value.ty(body, tcx);
-
-                let locations = term_location.to_locations();
-                if let Err(terr) =
-                    self.sub_types(rv_ty, place_ty, locations, ConstraintCategory::Assignment)
-                {
-                    span_mirbug!(
-                        self,
-                        term,
-                        "bad DropAndReplace ({:?} = {:?}): {:?}",
-                        place_ty,
-                        rv_ty,
-                        terr
-                    );
-                }
-            }
             TerminatorKind::SwitchInt { discr, .. } => {
                 self.check_operand(discr, term_location);
 
@@ -1357,20 +1339,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 };
                 let (sig, map) = tcx.replace_late_bound_regions(sig, |br| {
                     use crate::renumber::{BoundRegionInfo, RegionCtxt};
-                    use rustc_span::Symbol;
 
                     let region_ctxt_fn = || {
                         let reg_info = match br.kind {
-                            ty::BoundRegionKind::BrAnon(_, Some(span)) => {
-                                BoundRegionInfo::Span(span)
-                            }
-                            ty::BoundRegionKind::BrAnon(..) => {
-                                BoundRegionInfo::Name(Symbol::intern("anon"))
-                            }
+                            ty::BoundRegionKind::BrAnon(Some(span)) => BoundRegionInfo::Span(span),
+                            ty::BoundRegionKind::BrAnon(..) => BoundRegionInfo::Name(sym::anon),
                             ty::BoundRegionKind::BrNamed(_, name) => BoundRegionInfo::Name(name),
-                            ty::BoundRegionKind::BrEnv => {
-                                BoundRegionInfo::Name(Symbol::intern("env"))
-                            }
+                            ty::BoundRegionKind::BrEnv => BoundRegionInfo::Name(sym::env),
                         };
 
                         RegionCtxt::LateBound(reg_info)
@@ -1603,7 +1578,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     span_mirbug!(self, block_data, "resume on non-cleanup block!")
                 }
             }
-            TerminatorKind::Abort => {
+            TerminatorKind::Terminate => {
                 if !is_cleanup {
                     span_mirbug!(self, block_data, "abort on non-cleanup block!")
                 }
@@ -1629,26 +1604,15 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             }
             TerminatorKind::Unreachable => {}
             TerminatorKind::Drop { target, unwind, .. }
-            | TerminatorKind::DropAndReplace { target, unwind, .. }
-            | TerminatorKind::Assert { target, cleanup: unwind, .. } => {
+            | TerminatorKind::Assert { target, unwind, .. } => {
                 self.assert_iscleanup(body, block_data, target, is_cleanup);
-                if let Some(unwind) = unwind {
-                    if is_cleanup {
-                        span_mirbug!(self, block_data, "unwind on cleanup block")
-                    }
-                    self.assert_iscleanup(body, block_data, unwind, true);
-                }
+                self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
-            TerminatorKind::Call { ref target, cleanup, .. } => {
+            TerminatorKind::Call { ref target, unwind, .. } => {
                 if let &Some(target) = target {
                     self.assert_iscleanup(body, block_data, target, is_cleanup);
                 }
-                if let Some(cleanup) = cleanup {
-                    if is_cleanup {
-                        span_mirbug!(self, block_data, "cleanup on cleanup block")
-                    }
-                    self.assert_iscleanup(body, block_data, cleanup, true);
-                }
+                self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
             TerminatorKind::FalseEdge { real_target, imaginary_target } => {
                 self.assert_iscleanup(body, block_data, real_target, is_cleanup);
@@ -1656,23 +1620,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             }
             TerminatorKind::FalseUnwind { real_target, unwind } => {
                 self.assert_iscleanup(body, block_data, real_target, is_cleanup);
-                if let Some(unwind) = unwind {
-                    if is_cleanup {
-                        span_mirbug!(self, block_data, "cleanup in cleanup block via false unwind");
-                    }
-                    self.assert_iscleanup(body, block_data, unwind, true);
-                }
+                self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
-            TerminatorKind::InlineAsm { destination, cleanup, .. } => {
+            TerminatorKind::InlineAsm { destination, unwind, .. } => {
                 if let Some(target) = destination {
                     self.assert_iscleanup(body, block_data, target, is_cleanup);
                 }
-                if let Some(cleanup) = cleanup {
-                    if is_cleanup {
-                        span_mirbug!(self, block_data, "cleanup on cleanup block")
-                    }
-                    self.assert_iscleanup(body, block_data, cleanup, true);
-                }
+                self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
         }
     }
@@ -1689,6 +1643,29 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         }
     }
 
+    fn assert_iscleanup_unwind(
+        &mut self,
+        body: &Body<'tcx>,
+        ctxt: &dyn fmt::Debug,
+        unwind: UnwindAction,
+        is_cleanup: bool,
+    ) {
+        match unwind {
+            UnwindAction::Cleanup(unwind) => {
+                if is_cleanup {
+                    span_mirbug!(self, ctxt, "unwind on cleanup block")
+                }
+                self.assert_iscleanup(body, ctxt, unwind, true);
+            }
+            UnwindAction::Continue => {
+                if is_cleanup {
+                    span_mirbug!(self, ctxt, "unwind on cleanup block")
+                }
+            }
+            UnwindAction::Unreachable | UnwindAction::Terminate => (),
+        }
+    }
+
     fn check_local(&mut self, body: &Body<'tcx>, local: Local, local_decl: &LocalDecl<'tcx>) {
         match body.local_kind(local) {
             LocalKind::ReturnPointer | LocalKind::Arg => {
@@ -1700,7 +1677,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 // - maybe we should make that a warning.
                 return;
             }
-            LocalKind::Var | LocalKind::Temp => {}
+            LocalKind::Temp => {}
         }
 
         // When `unsized_fn_params` or `unsized_locals` is enabled, only function calls
@@ -1736,7 +1713,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     fn aggregate_field_ty(
         &mut self,
         ak: &AggregateKind<'tcx>,
-        field_index: usize,
+        field_index: FieldIdx,
         location: Location,
     ) -> Result<Ty<'tcx>, FieldAccessError> {
         let tcx = self.tcx();
@@ -1753,7 +1730,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 }
             }
             AggregateKind::Closure(_, substs) => {
-                match substs.as_closure().upvar_tys().nth(field_index) {
+                match substs.as_closure().upvar_tys().nth(field_index.as_usize()) {
                     Some(ty) => Ok(ty),
                     None => Err(FieldAccessError::OutOfRange {
                         field_count: substs.as_closure().upvar_tys().count(),
@@ -1764,7 +1741,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 // It doesn't make sense to look at a field beyond the prefix;
                 // these require a variant index, and are not initialized in
                 // aggregate rvalues.
-                match substs.as_generator().prefix_tys().nth(field_index) {
+                match substs.as_generator().prefix_tys().nth(field_index.as_usize()) {
                     Some(ty) => Ok(ty),
                     None => Err(FieldAccessError::OutOfRange {
                         field_count: substs.as_generator().prefix_tys().count(),
@@ -2242,6 +2219,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                             }
                         }
                     }
+                    CastKind::Transmute => {
+                        span_mirbug!(
+                            self,
+                            rvalue,
+                            "Unexpected CastKind::Transmute, which is not permitted in Analysis MIR",
+                        );
+                    }
                 }
             }
 
@@ -2362,7 +2346,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         body: &Body<'tcx>,
         rvalue: &Rvalue<'tcx>,
         aggregate_kind: &AggregateKind<'tcx>,
-        operands: &[Operand<'tcx>],
+        operands: &IndexSlice<FieldIdx, Operand<'tcx>>,
         location: Location,
     ) {
         let tcx = self.tcx();
@@ -2374,7 +2358,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             return;
         }
 
-        for (i, operand) in operands.iter().enumerate() {
+        for (i, operand) in operands.iter_enumerated() {
             let field_ty = match self.aggregate_field_ty(aggregate_kind, i, location) {
                 Ok(field_ty) => field_ty,
                 Err(FieldAccessError::OutOfRange { field_count }) => {
@@ -2382,8 +2366,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                         self,
                         rvalue,
                         "accessed field #{} but variant only has {}",
-                        i,
-                        field_count
+                        i.as_u32(),
+                        field_count,
                     );
                     continue;
                 }
@@ -2435,7 +2419,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         if let Some(all_facts) = all_facts {
             let _prof_timer = self.infcx.tcx.prof.generic_activity("polonius_fact_generation");
             if let Some(borrow_index) = borrow_set.get_index_of(&location) {
-                let region_vid = borrow_region.to_region_vid();
+                let region_vid = borrow_region.as_var();
                 all_facts.loan_issued_at.push((
                     region_vid,
                     borrow_index,
@@ -2481,8 +2465,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     match base_ty.kind() {
                         ty::Ref(ref_region, _, mutbl) => {
                             constraints.outlives_constraints.push(OutlivesConstraint {
-                                sup: ref_region.to_region_vid(),
-                                sub: borrow_region.to_region_vid(),
+                                sup: ref_region.as_var(),
+                                sub: borrow_region.as_var(),
                                 locations: location.to_locations(),
                                 span: location.to_locations().span(body),
                                 category,
@@ -2612,7 +2596,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 self.implicit_region_bound,
                 self.param_env,
                 location.to_locations(),
-                DUMMY_SP,                   // irrelevant; will be overrided.
+                DUMMY_SP,                   // irrelevant; will be overridden.
                 ConstraintCategory::Boring, // same as above.
                 &mut self.borrowck_context.constraints,
             )
diff --git a/compiler/rustc_borrowck/src/type_check/relate_tys.rs b/compiler/rustc_borrowck/src/type_check/relate_tys.rs
index d96372fb9..7e6d17ec3 100644
--- a/compiler/rustc_borrowck/src/type_check/relate_tys.rs
+++ b/compiler/rustc_borrowck/src/type_check/relate_tys.rs
@@ -4,6 +4,7 @@ use rustc_infer::traits::PredicateObligations;
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::ty::relate::TypeRelation;
 use rustc_middle::ty::{self, Ty};
+use rustc_span::symbol::sym;
 use rustc_span::{Span, Symbol};
 use rustc_trait_selection::traits::query::Fallible;
 
@@ -123,18 +124,17 @@ impl<'tcx> TypeRelatingDelegate<'tcx> for NllTypeRelatingDelegate<'_, '_, 'tcx>
             .constraints
             .placeholder_region(self.type_checker.infcx, placeholder);
 
-        let reg_info = match placeholder.name {
-            ty::BoundRegionKind::BrAnon(_, Some(span)) => BoundRegionInfo::Span(span),
-            ty::BoundRegionKind::BrAnon(..) => BoundRegionInfo::Name(Symbol::intern("anon")),
+        let reg_info = match placeholder.bound.kind {
+            ty::BoundRegionKind::BrAnon(Some(span)) => BoundRegionInfo::Span(span),
+            ty::BoundRegionKind::BrAnon(..) => BoundRegionInfo::Name(sym::anon),
             ty::BoundRegionKind::BrNamed(_, name) => BoundRegionInfo::Name(name),
-            ty::BoundRegionKind::BrEnv => BoundRegionInfo::Name(Symbol::intern("env")),
+            ty::BoundRegionKind::BrEnv => BoundRegionInfo::Name(sym::env),
         };
 
-        let reg_var =
-            reg.as_var().unwrap_or_else(|| bug!("expected region {:?} to be of kind ReVar", reg));
-        let mut var_to_origin = self.type_checker.infcx.reg_var_to_origin.borrow_mut();
-        let prev = var_to_origin.insert(reg_var, RegionCtxt::Placeholder(reg_info));
-        assert!(matches!(prev, None));
+        if cfg!(debug_assertions) && !self.type_checker.infcx.inside_canonicalization_ctxt() {
+            let mut var_to_origin = self.type_checker.infcx.reg_var_to_origin.borrow_mut();
+            var_to_origin.insert(reg.as_var(), RegionCtxt::Placeholder(reg_info));
+        }
 
         reg
     }
@@ -146,17 +146,9 @@ impl<'tcx> TypeRelatingDelegate<'tcx> for NllTypeRelatingDelegate<'_, '_, 'tcx>
             universe,
         );
 
-        let reg_var =
-            reg.as_var().unwrap_or_else(|| bug!("expected region {:?} to be of kind ReVar", reg));
-
-        if cfg!(debug_assertions) {
+        if cfg!(debug_assertions) && !self.type_checker.infcx.inside_canonicalization_ctxt() {
             let mut var_to_origin = self.type_checker.infcx.reg_var_to_origin.borrow_mut();
-            let prev = var_to_origin.insert(reg_var, RegionCtxt::Existential(None));
-
-            // It only makes sense to track region vars in non-canonicalization contexts. If this
-            // ever changes we either want to get rid of `BorrowckInferContext::reg_var_to_origin`
-            // or modify how we track nll region vars for that map.
-            assert!(matches!(prev, None));
+            var_to_origin.insert(reg.as_var(), RegionCtxt::Existential(None));
         }
 
         reg
diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs
index 15d7613a8..70fddb105 100644
--- a/compiler/rustc_borrowck/src/universal_regions.rs
+++ b/compiler/rustc_borrowck/src/universal_regions.rs
@@ -22,14 +22,12 @@ use rustc_hir::BodyOwnerKind;
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_infer::infer::NllRegionVariableOrigin;
 use rustc_middle::ty::fold::TypeFoldable;
-use rustc_middle::ty::{
-    self, DefIdTree, InlineConstSubsts, InlineConstSubstsParts, RegionVid, Ty, TyCtxt,
-};
+use rustc_middle::ty::{self, InlineConstSubsts, InlineConstSubstsParts, RegionVid, Ty, TyCtxt};
 use rustc_middle::ty::{InternalSubsts, SubstsRef};
+use rustc_span::symbol::{kw, sym};
 use rustc_span::Symbol;
 use std::iter;
 
-use crate::nll::ToRegionVid;
 use crate::renumber::{BoundRegionInfo, RegionCtxt};
 use crate::BorrowckInferCtxt;
 
@@ -314,6 +312,9 @@ impl<'tcx> UniversalRegions<'tcx> {
     }
 
     /// Gets an iterator over all the early-bound regions that have names.
+    /// Iteration order may be unstable, so this should only be used when
+    /// iteration order doesn't affect anything
+    #[allow(rustc::potential_query_instability)]
     pub fn named_universal_regions<'s>(
         &'s self,
     ) -> impl Iterator<Item = (ty::Region<'tcx>, ty::RegionVid)> + 's {
@@ -403,10 +404,8 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
         assert_eq!(FIRST_GLOBAL_INDEX, self.infcx.num_region_vars());
 
         // Create the "global" region that is always free in all contexts: 'static.
-        let fr_static = self
-            .infcx
-            .next_nll_region_var(FR, || RegionCtxt::Free(Symbol::intern("static")))
-            .to_region_vid();
+        let fr_static =
+            self.infcx.next_nll_region_var(FR, || RegionCtxt::Free(kw::Static)).as_var();
 
         // We've now added all the global regions. The next ones we
         // add will be external.
@@ -439,18 +438,14 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                     debug!(?r);
                     if !indices.indices.contains_key(&r) {
                         let region_vid = {
-                            let name = match r.get_name() {
-                                Some(name) => name,
-                                _ => Symbol::intern("anon"),
-                            };
-
+                            let name = r.get_name_or_anon();
                             self.infcx.next_nll_region_var(FR, || {
                                 RegionCtxt::LateBound(BoundRegionInfo::Name(name))
                             })
                         };
 
                         debug!(?region_vid);
-                        indices.insert_late_bound_region(r, region_vid.to_region_vid());
+                        indices.insert_late_bound_region(r, region_vid.as_var());
                     }
                 },
             );
@@ -477,18 +472,14 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
             debug!(?r);
             if !indices.indices.contains_key(&r) {
                 let region_vid = {
-                    let name = match r.get_name() {
-                        Some(name) => name,
-                        _ => Symbol::intern("anon"),
-                    };
-
+                    let name = r.get_name_or_anon();
                     self.infcx.next_nll_region_var(FR, || {
                         RegionCtxt::LateBound(BoundRegionInfo::Name(name))
                     })
                 };
 
                 debug!(?region_vid);
-                indices.insert_late_bound_region(r, region_vid.to_region_vid());
+                indices.insert_late_bound_region(r, region_vid.as_var());
             }
         });
 
@@ -507,7 +498,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                 let reg_vid = self
                     .infcx
                     .next_nll_region_var(FR, || RegionCtxt::Free(Symbol::intern("c-variadic")))
-                    .to_region_vid();
+                    .as_var();
 
                 let region = self.infcx.tcx.mk_re_var(reg_vid);
                 let va_list_ty =
@@ -522,7 +513,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
         let fr_fn_body = self
             .infcx
             .next_nll_region_var(FR, || RegionCtxt::Free(Symbol::intern("fn_body")))
-            .to_region_vid();
+            .as_var();
 
         let num_universals = self.infcx.num_region_vars();
 
@@ -643,7 +634,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
 
         let global_mapping = iter::once((tcx.lifetimes.re_static, fr_static));
         let subst_mapping =
-            iter::zip(identity_substs.regions(), fr_substs.regions().map(|r| r.to_region_vid()));
+            iter::zip(identity_substs.regions(), fr_substs.regions().map(|r| r.as_var()));
 
         UniversalRegionIndices { indices: global_mapping.chain(subst_mapping).collect(), fr_static }
     }
@@ -767,15 +758,10 @@ impl<'cx, 'tcx> InferCtxtExt<'tcx> for BorrowckInferCtxt<'cx, 'tcx> {
         T: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.infcx.tcx.fold_regions(value, |region, _depth| {
-            let name = match region.get_name() {
-                Some(name) => name,
-                _ => Symbol::intern("anon"),
-            };
+            let name = region.get_name_or_anon();
             debug!(?region, ?name);
 
-            let reg_var = self.next_nll_region_var(origin, || RegionCtxt::Free(name));
-
-            reg_var
+            self.next_nll_region_var(origin, || RegionCtxt::Free(name))
         })
     }
 
@@ -796,13 +782,13 @@ impl<'cx, 'tcx> InferCtxtExt<'tcx> for BorrowckInferCtxt<'cx, 'tcx> {
             let region_vid = {
                 let name = match br.kind.get_name() {
                     Some(name) => name,
-                    _ => Symbol::intern("anon"),
+                    _ => sym::anon,
                 };
 
                 self.next_nll_region_var(origin, || RegionCtxt::Bound(BoundRegionInfo::Name(name)))
             };
 
-            indices.insert_late_bound_region(liberated_region, region_vid.to_region_vid());
+            indices.insert_late_bound_region(liberated_region, region_vid.as_var());
             debug!(?liberated_region, ?region_vid);
             region_vid
         });
@@ -828,18 +814,14 @@ impl<'cx, 'tcx> InferCtxtExt<'tcx> for BorrowckInferCtxt<'cx, 'tcx> {
             debug!(?r);
             if !indices.indices.contains_key(&r) {
                 let region_vid = {
-                    let name = match r.get_name() {
-                        Some(name) => name,
-                        _ => Symbol::intern("anon"),
-                    };
-
+                    let name = r.get_name_or_anon();
                     self.next_nll_region_var(FR, || {
                         RegionCtxt::LateBound(BoundRegionInfo::Name(name))
                     })
                 };
 
                 debug!(?region_vid);
-                indices.insert_late_bound_region(r, region_vid.to_region_vid());
+                indices.insert_late_bound_region(r, region_vid.as_var());
             }
         });
     }
@@ -854,17 +836,13 @@ impl<'cx, 'tcx> InferCtxtExt<'tcx> for BorrowckInferCtxt<'cx, 'tcx> {
             debug!(?r);
             if !indices.indices.contains_key(&r) {
                 let region_vid = {
-                    let name = match r.get_name() {
-                        Some(name) => name,
-                        _ => Symbol::intern("anon"),
-                    };
-
+                    let name = r.get_name_or_anon();
                     self.next_nll_region_var(FR, || {
                         RegionCtxt::LateBound(BoundRegionInfo::Name(name))
                     })
                 };
 
-                indices.insert_late_bound_region(r, region_vid.to_region_vid());
+                indices.insert_late_bound_region(r, region_vid.as_var());
             }
         });
     }
@@ -882,7 +860,7 @@ impl<'tcx> UniversalRegionIndices<'tcx> {
     }
 
     /// Converts `r` into a local inference variable: `r` can either
-    /// by a `ReVar` (i.e., already a reference to an inference
+    /// be a `ReVar` (i.e., already a reference to an inference
     /// variable) or it can be `'static` or some early-bound
     /// region. This is useful when taking the results from
     /// type-checking and trait-matching, which may sometimes
@@ -891,7 +869,7 @@ impl<'tcx> UniversalRegionIndices<'tcx> {
     /// fully initialized.
     pub fn to_region_vid(&self, r: ty::Region<'tcx>) -> RegionVid {
         if let ty::ReVar(..) = *r {
-            r.to_region_vid()
+            r.as_var()
         } else if r.is_error() {
             // We use the `'static` `RegionVid` because `ReError` doesn't actually exist in the
             // `UniversalRegionIndices`. This is fine because 1) it is a fallback only used if
diff --git a/compiler/rustc_borrowck/src/used_muts.rs b/compiler/rustc_borrowck/src/used_muts.rs
index e297b1230..c5991e0bc 100644
--- a/compiler/rustc_borrowck/src/used_muts.rs
+++ b/compiler/rustc_borrowck/src/used_muts.rs
@@ -1,6 +1,6 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_middle::mir::visit::{PlaceContext, Visitor};
 use rustc_middle::mir::{
     Local, Location, Place, Statement, StatementKind, Terminator, TerminatorKind,
@@ -26,8 +26,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
     ///  See #55344 for context.
     pub(crate) fn gather_used_muts(
         &mut self,
-        temporary_used_locals: FxHashSet<Local>,
-        mut never_initialized_mut_locals: FxHashSet<Local>,
+        temporary_used_locals: FxIndexSet<Local>,
+        mut never_initialized_mut_locals: FxIndexSet<Local>,
     ) {
         {
             let mut visitor = GatherUsedMutsVisitor {
@@ -48,8 +48,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 /// MIR visitor for collecting used mutable variables.
 /// The 'visit lifetime represents the duration of the MIR walk.
 struct GatherUsedMutsVisitor<'visit, 'cx, 'tcx> {
-    temporary_used_locals: FxHashSet<Local>,
-    never_initialized_mut_locals: &'visit mut FxHashSet<Local>,
+    temporary_used_locals: FxIndexSet<Local>,
+    never_initialized_mut_locals: &'visit mut FxIndexSet<Local>,
     mbcx: &'visit mut MirBorrowckCtxt<'cx, 'tcx>,
 }
 
@@ -71,9 +71,6 @@ impl<'visit, 'cx, 'tcx> Visitor<'tcx> for GatherUsedMutsVisitor<'visit, 'cx, 'tc
             TerminatorKind::Call { destination, .. } => {
                 self.remove_never_initialized_mut_locals(*destination);
             }
-            TerminatorKind::DropAndReplace { place, .. } => {
-                self.remove_never_initialized_mut_locals(*place);
-            }
             _ => {}
         }
 
diff --git a/compiler/rustc_builtin_macros/locales/en-US.ftl b/compiler/rustc_builtin_macros/locales/en-US.ftl
deleted file mode 100644
index 4d088e27b..000000000
--- a/compiler/rustc_builtin_macros/locales/en-US.ftl
+++ /dev/null
@@ -1,5 +0,0 @@
-builtin_macros_requires_cfg_pattern =
-    macro requires a cfg-pattern as an argument
-    .label = cfg-pattern required
-
-builtin_macros_expected_one_cfg_pattern = expected 1 cfg-pattern
diff --git a/compiler/rustc_builtin_macros/messages.ftl b/compiler/rustc_builtin_macros/messages.ftl
new file mode 100644
index 000000000..83dc1ac50
--- /dev/null
+++ b/compiler/rustc_builtin_macros/messages.ftl
@@ -0,0 +1,151 @@
+builtin_macros_requires_cfg_pattern =
+    macro requires a cfg-pattern as an argument
+    .label = cfg-pattern required
+
+builtin_macros_expected_one_cfg_pattern = expected 1 cfg-pattern
+
+builtin_macros_alloc_error_must_be_fn = alloc_error_handler must be a function
+
+builtin_macros_assert_requires_boolean = macro requires a boolean expression as an argument
+    .label = boolean expression required
+
+builtin_macros_assert_requires_expression = macro requires an expression as an argument
+    .suggestion = try removing semicolon
+
+builtin_macros_assert_missing_comma = unexpected string literal
+    .suggestion = try adding a comma
+
+builtin_macros_cfg_accessible_unspecified_path = `cfg_accessible` path is not specified
+builtin_macros_cfg_accessible_multiple_paths = multiple `cfg_accessible` paths are specified
+builtin_macros_cfg_accessible_literal_path = `cfg_accessible` path cannot be a literal
+builtin_macros_cfg_accessible_has_args = `cfg_accessible` path cannot accept arguments
+
+builtin_macros_cfg_accessible_indeterminate = cannot determine whether the path is accessible or not
+
+builtin_macros_concat_bytestr = cannot concatenate a byte string literal
+
+builtin_macros_concat_missing_literal = expected a literal
+    .note = only literals (like `"foo"`, `-42` and `3.14`) can be passed to `concat!()`
+
+builtin_macros_concat_bytes_missing_literal = expected a byte literal
+    .note = only byte literals (like `b"foo"`, `b's'` and `[3, 4, 5]`) can be passed to `concat_bytes!()`
+
+builtin_macros_concat_bytes_invalid = cannot concatenate {$lit_kind} literals
+    .byte_char = try using a byte character
+    .byte_str = try using a byte string
+    .number_array = try wrapping the number in an array
+
+builtin_macros_concat_bytes_oob = numeric literal is out of bounds
+
+builtin_macros_concat_bytes_non_u8 = numeric literal is not a `u8`
+
+builtin_macros_concat_bytes_array = cannot concatenate doubly nested array
+    .note = byte strings are treated as arrays of bytes
+    .help = try flattening the array
+
+builtin_macros_concat_bytes_bad_repeat = repeat count is not a positive number
+
+builtin_macros_concat_idents_missing_args = `concat_idents!()` takes 1 or more arguments
+builtin_macros_concat_idents_missing_comma = `concat_idents!()` expecting comma
+builtin_macros_concat_idents_ident_args = `concat_idents!()` requires ident args
+
+builtin_macros_bad_derive_target = `derive` may only be applied to `struct`s, `enum`s and `union`s
+    .label = not applicable here
+    .label2 = not a `struct`, `enum` or `union`
+
+builtin_macros_unexpected_lit = expected path to a trait, found literal
+    .label = not a trait
+    .str_lit = try using `#[derive({$sym})]`
+    .other = for example, write `#[derive(Debug)]` for `Debug`
+
+builtin_macros_derive_path_args_list = traits in `#[derive(...)]` don't accept arguments
+    .suggestion = remove the arguments
+
+builtin_macros_derive_path_args_value = traits in `#[derive(...)]` don't accept values
+    .suggestion = remove the value
+
+builtin_macros_derive_macro_call = `derive` cannot be used on items with type macros
+
+builtin_macros_cannot_derive_union = this trait cannot be derived for unions
+
+builtin_macros_no_default_variant = no default declared
+    .help = make a unit variant default by placing `#[default]` above it
+    .suggestion = make `{$ident}` default
+
+builtin_macros_multiple_defaults = multiple declared defaults
+    .label = first default
+    .additional = additional default
+    .note = only one variant can be default
+    .suggestion = make `{$ident}` default
+
+builtin_macros_non_unit_default = the `#[default]` attribute may only be used on unit enum variants
+    .help = consider a manual implementation of `Default`
+
+builtin_macros_non_exhaustive_default = default variant must be exhaustive
+    .label = declared `#[non_exhaustive]` here
+    .help = consider a manual implementation of `Default`
+
+builtin_macros_multiple_default_attrs = multiple `#[default]` attributes
+    .note = only one `#[default]` attribute is needed
+    .label = `#[default]` used here
+    .label_again = `#[default]` used again here
+    .help = try removing {$only_one ->
+    [true] this
+    *[false] these
+    }
+
+builtin_macros_default_arg = `#[default]` attribute does not accept a value
+    .suggestion = try using `#[default]`
+
+builtin_macros_env_takes_args = `env!()` takes 1 or 2 arguments
+
+builtin_macros_env_not_defined = environment variable `{$var}` not defined at compile time
+    .cargo = Cargo sets build script variables at run time. Use `std::env::var("{$var}")` instead
+    .other = use `std::env::var("{$var}")` to read the variable at run time
+
+builtin_macros_format_requires_string = requires at least a format string argument
+
+builtin_macros_format_duplicate_arg = duplicate argument named `{$ident}`
+    .label1 = previously here
+    .label2 = duplicate argument
+
+builtin_macros_format_positional_after_named = positional arguments cannot follow named arguments
+    .label = positional arguments must be before named arguments
+    .named_args = named argument
+
+builtin_macros_format_string_invalid = invalid format string: {$desc}
+    .label = {$label1} in format string
+    .note = {$note}
+    .second_label = {$label}
+
+builtin_macros_sugg = consider using a positional formatting argument instead
+
+builtin_macros_format_no_arg_named = there is no argument named `{$name}`
+    .note = did you intend to capture a variable `{$name}` from the surrounding scope?
+    .note2 = to avoid ambiguity, `format_args!` cannot capture variables when the format string is expanded from a macro
+
+builtin_macros_format_unknown_trait = unknown format trait `{$ty}`
+    .note = the only appropriate formatting traits are:
+                                            - ``, which uses the `Display` trait
+                                            - `?`, which uses the `Debug` trait
+                                            - `e`, which uses the `LowerExp` trait
+                                            - `E`, which uses the `UpperExp` trait
+                                            - `o`, which uses the `Octal` trait
+                                            - `p`, which uses the `Pointer` trait
+                                            - `b`, which uses the `Binary` trait
+                                            - `x`, which uses the `LowerHex` trait
+                                            - `X`, which uses the `UpperHex` trait
+    .suggestion = use the `{$trait_name}` trait
+
+builtin_macros_format_unused_arg = {$named ->
+    [true] named argument
+    *[false] argument
+    } never used
+
+builtin_macros_format_unused_args = multiple unused formatting arguments
+    .label = multiple missing formatting specifiers
+
+builtin_macros_format_pos_mismatch = {$n} positional {$n ->
+    [one] argument
+    *[more] arguments
+    } in format string, but {$desc}
diff --git a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
index ac6697232..82bae9157 100644
--- a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
+++ b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
@@ -1,3 +1,4 @@
+use crate::errors;
 use crate::util::check_builtin_macro_attribute;
 
 use rustc_ast::ptr::P;
@@ -31,7 +32,7 @@ pub fn expand(
         {
             (item, true, ecx.with_def_site_ctxt(fn_kind.sig.span))
         } else {
-            ecx.sess.parse_sess.span_diagnostic.span_err(item.span(), "alloc_error_handler must be a function");
+            ecx.sess.parse_sess.span_diagnostic.emit_err(errors::AllocErrorMustBeFn {span: item.span() });
             return vec![orig_item];
         };
 
diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs
index 3fdbc9715..8c1579baa 100644
--- a/compiler/rustc_builtin_macros/src/asm.rs
+++ b/compiler/rustc_builtin_macros/src/asm.rs
@@ -203,17 +203,6 @@ pub fn parse_asm_args<'a>(
         // Validate the order of named, positional & explicit register operands and
         // clobber_abi/options. We do this at the end once we have the full span
         // of the argument available.
-        if !args.options_spans.is_empty() {
-            diag.struct_span_err(span, "arguments are not allowed after options")
-                .span_labels(args.options_spans.clone(), "previous options")
-                .span_label(span, "argument")
-                .emit();
-        } else if let Some((_, abi_span)) = args.clobber_abis.last() {
-            diag.struct_span_err(span, "arguments are not allowed after clobber_abi")
-                .span_label(*abi_span, "clobber_abi")
-                .span_label(span, "argument")
-                .emit();
-        }
         if explicit_reg {
             if name.is_some() {
                 diag.struct_span_err(span, "explicit register arguments cannot have names").emit();
@@ -227,17 +216,6 @@ pub fn parse_asm_args<'a>(
                     .emit();
                 continue;
             }
-            if !args.reg_args.is_empty() {
-                let mut err = diag.struct_span_err(
-                    span,
-                    "named arguments cannot follow explicit register arguments",
-                );
-                err.span_label(span, "named argument");
-                for pos in &args.reg_args {
-                    err.span_label(args.operands[*pos].1, "explicit register argument");
-                }
-                err.emit();
-            }
             args.named_args.insert(name, slot);
         } else {
             if !args.named_args.is_empty() || !args.reg_args.is_empty() {
@@ -478,15 +456,6 @@ fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a,
 
     let full_span = span_start.to(p.prev_token.span);
 
-    if !args.options_spans.is_empty() {
-        let mut err = p
-            .sess
-            .span_diagnostic
-            .struct_span_err(full_span, "clobber_abi is not allowed after options");
-        err.span_labels(args.options_spans.clone(), "options");
-        return Err(err);
-    }
-
     match &new_abis[..] {
         // should have errored above during parsing
         [] => unreachable!(),
@@ -699,6 +668,10 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, args: AsmArgs) -> Option<ast::Inl
                                         args.operands[idx].1,
                                         "explicit register arguments cannot be used in the asm template",
                                     );
+                                    err.span_help(
+                                        args.operands[idx].1,
+                                        "use the register name directly in the assembly code",
+                                    );
                                 }
                                 err.emit();
                                 None
diff --git a/compiler/rustc_builtin_macros/src/assert.rs b/compiler/rustc_builtin_macros/src/assert.rs
index 75af5e2b1..0de424be2 100644
--- a/compiler/rustc_builtin_macros/src/assert.rs
+++ b/compiler/rustc_builtin_macros/src/assert.rs
@@ -1,12 +1,13 @@
 mod context;
 
 use crate::edition_panic::use_panic_2021;
+use crate::errors;
 use rustc_ast::ptr::P;
 use rustc_ast::token;
 use rustc_ast::tokenstream::{DelimSpan, TokenStream};
 use rustc_ast::{DelimArgs, Expr, ExprKind, MacCall, MacDelimiter, Path, PathSegment, UnOp};
 use rustc_ast_pretty::pprust;
-use rustc_errors::{Applicability, PResult};
+use rustc_errors::PResult;
 use rustc_expand::base::{DummyResult, ExtCtxt, MacEager, MacResult};
 use rustc_parse::parser::Parser;
 use rustc_span::symbol::{sym, Ident, Symbol};
@@ -114,9 +115,7 @@ fn parse_assert<'a>(cx: &mut ExtCtxt<'a>, sp: Span, stream: TokenStream) -> PRes
     let mut parser = cx.new_parser_from_tts(stream);
 
     if parser.token == token::Eof {
-        let mut err = cx.struct_span_err(sp, "macro requires a boolean expression as an argument");
-        err.span_label(sp, "boolean expression required");
-        return Err(err);
+        return Err(cx.create_err(errors::AssertRequiresBoolean { span: sp }));
     }
 
     let cond_expr = parser.parse_expr()?;
@@ -129,15 +128,7 @@ fn parse_assert<'a>(cx: &mut ExtCtxt<'a>, sp: Span, stream: TokenStream) -> PRes
     //
     // Emit an error about semicolon and suggest removing it.
     if parser.token == token::Semi {
-        let mut err = cx.struct_span_err(sp, "macro requires an expression as an argument");
-        err.span_suggestion(
-            parser.token.span,
-            "try removing semicolon",
-            "",
-            Applicability::MaybeIncorrect,
-        );
-        err.emit();
-
+        cx.emit_err(errors::AssertRequiresExpression { span: sp, token: parser.token.span });
         parser.bump();
     }
 
@@ -149,15 +140,8 @@ fn parse_assert<'a>(cx: &mut ExtCtxt<'a>, sp: Span, stream: TokenStream) -> PRes
     // Emit an error and suggest inserting a comma.
     let custom_message =
         if let token::Literal(token::Lit { kind: token::Str, .. }) = parser.token.kind {
-            let mut err = cx.struct_span_err(parser.token.span, "unexpected string literal");
-            let comma_span = parser.prev_token.span.shrink_to_hi();
-            err.span_suggestion_short(
-                comma_span,
-                "try adding a comma",
-                ", ",
-                Applicability::MaybeIncorrect,
-            );
-            err.emit();
+            let comma = parser.prev_token.span.shrink_to_hi();
+            cx.emit_err(errors::AssertMissingComma { span: parser.token.span, comma });
 
             parse_custom_message(&mut parser)
         } else if parser.eat(&token::Comma) {
diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs
index b0b4dda16..c9e3cd486 100644
--- a/compiler/rustc_builtin_macros/src/assert/context.rs
+++ b/compiler/rustc_builtin_macros/src/assert/context.rs
@@ -287,10 +287,9 @@ impl<'cx, 'a> Context<'cx, 'a> {
             // sync with the `rfc-2011-nicer-assert-messages/all-expr-kinds.rs` test.
             ExprKind::Assign(_, _, _)
             | ExprKind::AssignOp(_, _, _)
-            | ExprKind::Async(_, _, _)
+            | ExprKind::Async(_, _)
             | ExprKind::Await(_)
             | ExprKind::Block(_, _)
-            | ExprKind::Box(_)
             | ExprKind::Break(_, _)
             | ExprKind::Closure(_)
             | ExprKind::ConstBlock(_)
diff --git a/compiler/rustc_builtin_macros/src/cfg.rs b/compiler/rustc_builtin_macros/src/cfg.rs
index 5638c2f61..1397cee7a 100644
--- a/compiler/rustc_builtin_macros/src/cfg.rs
+++ b/compiler/rustc_builtin_macros/src/cfg.rs
@@ -2,13 +2,13 @@
 //! a literal `true` or `false` based on whether the given cfg matches the
 //! current compilation environment.
 
+use crate::errors;
 use rustc_ast as ast;
 use rustc_ast::token;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_attr as attr;
 use rustc_errors::PResult;
 use rustc_expand::base::{self, *};
-use rustc_macros::Diagnostic;
 use rustc_span::Span;
 
 pub fn expand_cfg(
@@ -35,26 +35,11 @@ pub fn expand_cfg(
     }
 }
 
-#[derive(Diagnostic)]
-#[diag(builtin_macros_requires_cfg_pattern)]
-struct RequiresCfgPattern {
-    #[primary_span]
-    #[label]
-    span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(builtin_macros_expected_one_cfg_pattern)]
-struct OneCfgPattern {
-    #[primary_span]
-    span: Span,
-}
-
 fn parse_cfg<'a>(cx: &mut ExtCtxt<'a>, span: Span, tts: TokenStream) -> PResult<'a, ast::MetaItem> {
     let mut p = cx.new_parser_from_tts(tts);
 
     if p.token == token::Eof {
-        return Err(cx.create_err(RequiresCfgPattern { span }));
+        return Err(cx.create_err(errors::RequiresCfgPattern { span }));
     }
 
     let cfg = p.parse_meta_item()?;
@@ -62,7 +47,7 @@ fn parse_cfg<'a>(cx: &mut ExtCtxt<'a>, span: Span, tts: TokenStream) -> PResult<
     let _ = p.eat(&token::Comma);
 
     if !p.eat(&token::Eof) {
-        return Err(cx.create_err(OneCfgPattern { span }));
+        return Err(cx.create_err(errors::OneCfgPattern { span }));
     }
 
     Ok(cfg)
diff --git a/compiler/rustc_builtin_macros/src/cfg_accessible.rs b/compiler/rustc_builtin_macros/src/cfg_accessible.rs
index 4e4cafc71..37ac09ccd 100644
--- a/compiler/rustc_builtin_macros/src/cfg_accessible.rs
+++ b/compiler/rustc_builtin_macros/src/cfg_accessible.rs
@@ -1,5 +1,6 @@
 //! Implementation of the `#[cfg_accessible(path)]` attribute macro.
 
+use crate::errors;
 use rustc_ast as ast;
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier};
 use rustc_feature::AttributeTemplate;
@@ -10,15 +11,22 @@ use rustc_span::Span;
 pub(crate) struct Expander;
 
 fn validate_input<'a>(ecx: &mut ExtCtxt<'_>, mi: &'a ast::MetaItem) -> Option<&'a ast::Path> {
+    use errors::CfgAccessibleInvalid::*;
     match mi.meta_item_list() {
         None => {}
-        Some([]) => ecx.span_err(mi.span, "`cfg_accessible` path is not specified"),
-        Some([_, .., l]) => ecx.span_err(l.span(), "multiple `cfg_accessible` paths are specified"),
+        Some([]) => {
+            ecx.emit_err(UnspecifiedPath(mi.span));
+        }
+        Some([_, .., l]) => {
+            ecx.emit_err(MultiplePaths(l.span()));
+        }
         Some([nmi]) => match nmi.meta_item() {
-            None => ecx.span_err(nmi.span(), "`cfg_accessible` path cannot be a literal"),
+            None => {
+                ecx.emit_err(LiteralPath(nmi.span()));
+            }
             Some(mi) => {
                 if !mi.is_word() {
-                    ecx.span_err(mi.span, "`cfg_accessible` path cannot accept arguments");
+                    ecx.emit_err(HasArguments(mi.span));
                 }
                 return Some(&mi.path);
             }
@@ -53,7 +61,7 @@ impl MultiItemModifier for Expander {
             Ok(true) => ExpandResult::Ready(vec![item]),
             Ok(false) => ExpandResult::Ready(Vec::new()),
             Err(Indeterminate) if ecx.force_mode => {
-                ecx.span_err(span, "cannot determine whether the path is accessible or not");
+                ecx.emit_err(errors::CfgAccessibleIndeterminate { span });
                 ExpandResult::Ready(vec![item])
             }
             Err(Indeterminate) => ExpandResult::Retry(item),
diff --git a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs
index db05c00d2..2b6fcc169 100644
--- a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs
+++ b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs
@@ -6,7 +6,7 @@ use rustc_ast::{self as ast, AttrItem, AttrStyle};
 use rustc_session::parse::ParseSess;
 use rustc_span::FileName;
 
-pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -> ast::Crate {
+pub fn inject(krate: &mut ast::Crate, parse_sess: &ParseSess, attrs: &[String]) {
     for raw_attr in attrs {
         let mut parser = rustc_parse::new_parser_from_source_str(
             parse_sess,
@@ -36,6 +36,4 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
             start_span.to(end_span),
         ));
     }
-
-    krate
 }
diff --git a/compiler/rustc_builtin_macros/src/compile_error.rs b/compiler/rustc_builtin_macros/src/compile_error.rs
index 72397aa25..aeb3bb800 100644
--- a/compiler/rustc_builtin_macros/src/compile_error.rs
+++ b/compiler/rustc_builtin_macros/src/compile_error.rs
@@ -13,6 +13,11 @@ pub fn expand_compile_error<'cx>(
         return DummyResult::any(sp);
     };
 
+    #[expect(
+        rustc::diagnostic_outside_of_impl,
+        reason = "diagnostic message is specified by user"
+    )]
+    #[expect(rustc::untranslatable_diagnostic, reason = "diagnostic message is specified by user")]
     cx.span_err(sp, var.as_str());
 
     DummyResult::any(sp)
diff --git a/compiler/rustc_builtin_macros/src/concat.rs b/compiler/rustc_builtin_macros/src/concat.rs
index 7da9bdc38..b92964d03 100644
--- a/compiler/rustc_builtin_macros/src/concat.rs
+++ b/compiler/rustc_builtin_macros/src/concat.rs
@@ -4,6 +4,8 @@ use rustc_expand::base::{self, DummyResult};
 use rustc_session::errors::report_lit_error;
 use rustc_span::symbol::Symbol;
 
+use crate::errors;
+
 pub fn expand_concat(
     cx: &mut base::ExtCtxt<'_>,
     sp: rustc_span::Span,
@@ -31,7 +33,7 @@ pub fn expand_concat(
                     accumulator.push_str(&b.to_string());
                 }
                 Ok(ast::LitKind::Byte(..) | ast::LitKind::ByteStr(..)) => {
-                    cx.span_err(e.span, "cannot concatenate a byte string literal");
+                    cx.emit_err(errors::ConcatBytestr { span: e.span });
                     has_errors = true;
                 }
                 Ok(ast::LitKind::Err) => {
@@ -42,8 +44,20 @@ pub fn expand_concat(
                     has_errors = true;
                 }
             },
+            // We also want to allow negative numeric literals.
+            ast::ExprKind::Unary(ast::UnOp::Neg, ref expr) if let ast::ExprKind::Lit(token_lit) = expr.kind => {
+                match ast::LitKind::from_token_lit(token_lit) {
+                    Ok(ast::LitKind::Int(i, _)) => accumulator.push_str(&format!("-{i}")),
+                    Ok(ast::LitKind::Float(f, _)) => accumulator.push_str(&format!("-{f}")),
+                    Err(err) => {
+                        report_lit_error(&cx.sess.parse_sess, err, token_lit, e.span);
+                        has_errors = true;
+                    }
+                    _ => missing_literal.push(e.span),
+                }
+            }
             ast::ExprKind::IncludedBytes(..) => {
-                cx.span_err(e.span, "cannot concatenate a byte string literal")
+                cx.emit_err(errors::ConcatBytestr { span: e.span });
             }
             ast::ExprKind::Err => {
                 has_errors = true;
@@ -53,10 +67,9 @@ pub fn expand_concat(
             }
         }
     }
+
     if !missing_literal.is_empty() {
-        let mut err = cx.struct_span_err(missing_literal, "expected a literal");
-        err.note("only literals (like `\"foo\"`, `42` and `3.14`) can be passed to `concat!()`");
-        err.emit();
+        cx.emit_err(errors::ConcatMissingLiteral { spans: missing_literal });
         return DummyResult::any(sp);
     } else if has_errors {
         return DummyResult::any(sp);
diff --git a/compiler/rustc_builtin_macros/src/concat_bytes.rs b/compiler/rustc_builtin_macros/src/concat_bytes.rs
index 4f1a7d709..ba639c0a9 100644
--- a/compiler/rustc_builtin_macros/src/concat_bytes.rs
+++ b/compiler/rustc_builtin_macros/src/concat_bytes.rs
@@ -1,10 +1,11 @@
 use rustc_ast as ast;
 use rustc_ast::{ptr::P, tokenstream::TokenStream};
-use rustc_errors::Applicability;
 use rustc_expand::base::{self, DummyResult};
 use rustc_session::errors::report_lit_error;
 use rustc_span::Span;
 
+use crate::errors;
+
 /// Emits errors for literal expressions that are invalid inside and outside of an array.
 fn invalid_type_err(
     cx: &mut base::ExtCtxt<'_>,
@@ -12,62 +13,46 @@ fn invalid_type_err(
     span: Span,
     is_nested: bool,
 ) {
+    use errors::{
+        ConcatBytesInvalid, ConcatBytesInvalidSuggestion, ConcatBytesNonU8, ConcatBytesOob,
+    };
+    let snippet = cx.sess.source_map().span_to_snippet(span).ok();
     match ast::LitKind::from_token_lit(token_lit) {
         Ok(ast::LitKind::Char(_)) => {
-            let mut err = cx.struct_span_err(span, "cannot concatenate character literals");
-            if let Ok(snippet) = cx.sess.source_map().span_to_snippet(span) {
-                err.span_suggestion(
-                    span,
-                    "try using a byte character",
-                    format!("b{}", snippet),
-                    Applicability::MachineApplicable,
-                )
-                .emit();
-            }
+            let sugg =
+                snippet.map(|snippet| ConcatBytesInvalidSuggestion::CharLit { span, snippet });
+            cx.sess.emit_err(ConcatBytesInvalid { span, lit_kind: "character", sugg });
         }
         Ok(ast::LitKind::Str(_, _)) => {
-            let mut err = cx.struct_span_err(span, "cannot concatenate string literals");
             // suggestion would be invalid if we are nested
-            if !is_nested {
-                if let Ok(snippet) = cx.sess.source_map().span_to_snippet(span) {
-                    err.span_suggestion(
-                        span,
-                        "try using a byte string",
-                        format!("b{}", snippet),
-                        Applicability::MachineApplicable,
-                    );
-                }
-            }
-            err.emit();
+            let sugg = if !is_nested {
+                snippet.map(|snippet| ConcatBytesInvalidSuggestion::StrLit { span, snippet })
+            } else {
+                None
+            };
+            cx.emit_err(ConcatBytesInvalid { span, lit_kind: "string", sugg });
         }
         Ok(ast::LitKind::Float(_, _)) => {
-            cx.span_err(span, "cannot concatenate float literals");
+            cx.emit_err(ConcatBytesInvalid { span, lit_kind: "float", sugg: None });
         }
         Ok(ast::LitKind::Bool(_)) => {
-            cx.span_err(span, "cannot concatenate boolean literals");
+            cx.emit_err(ConcatBytesInvalid { span, lit_kind: "boolean", sugg: None });
         }
         Ok(ast::LitKind::Err) => {}
         Ok(ast::LitKind::Int(_, _)) if !is_nested => {
-            let mut err = cx.struct_span_err(span, "cannot concatenate numeric literals");
-            if let Ok(snippet) = cx.sess.source_map().span_to_snippet(span) {
-                err.span_suggestion(
-                    span,
-                    "try wrapping the number in an array",
-                    format!("[{}]", snippet),
-                    Applicability::MachineApplicable,
-                );
-            }
-            err.emit();
+            let sugg =
+                snippet.map(|snippet| ConcatBytesInvalidSuggestion::IntLit { span: span, snippet });
+            cx.emit_err(ConcatBytesInvalid { span, lit_kind: "numeric", sugg });
         }
         Ok(ast::LitKind::Int(
             val,
             ast::LitIntType::Unsuffixed | ast::LitIntType::Unsigned(ast::UintTy::U8),
         )) => {
             assert!(val > u8::MAX.into()); // must be an error
-            cx.span_err(span, "numeric literal is out of bounds");
+            cx.emit_err(ConcatBytesOob { span });
         }
         Ok(ast::LitKind::Int(_, _)) => {
-            cx.span_err(span, "numeric literal is not a `u8`");
+            cx.emit_err(ConcatBytesNonU8 { span });
         }
         Ok(ast::LitKind::ByteStr(..) | ast::LitKind::Byte(_)) => unreachable!(),
         Err(err) => {
@@ -85,7 +70,7 @@ fn handle_array_element(
     match expr.kind {
         ast::ExprKind::Array(_) | ast::ExprKind::Repeat(_, _) => {
             if !*has_errors {
-                cx.span_err(expr.span, "cannot concatenate doubly nested array");
+                cx.emit_err(errors::ConcatBytesArray { span: expr.span, bytestr: false });
             }
             *has_errors = true;
             None
@@ -99,10 +84,7 @@ fn handle_array_element(
             Ok(ast::LitKind::Byte(val)) => Some(val),
             Ok(ast::LitKind::ByteStr(..)) => {
                 if !*has_errors {
-                    cx.struct_span_err(expr.span, "cannot concatenate doubly nested array")
-                        .note("byte strings are treated as arrays of bytes")
-                        .help("try flattening the array")
-                        .emit();
+                    cx.emit_err(errors::ConcatBytesArray { span: expr.span, bytestr: true });
                 }
                 *has_errors = true;
                 None
@@ -117,10 +99,7 @@ fn handle_array_element(
         },
         ast::ExprKind::IncludedBytes(..) => {
             if !*has_errors {
-                cx.struct_span_err(expr.span, "cannot concatenate doubly nested array")
-                    .note("byte strings are treated as arrays of bytes")
-                    .help("try flattening the array")
-                    .emit();
+                cx.emit_err(errors::ConcatBytesArray { span: expr.span, bytestr: false });
             }
             *has_errors = true;
             None
@@ -167,7 +146,7 @@ pub fn expand_concat_bytes(
                         }
                     }
                 } else {
-                    cx.span_err(count.value.span, "repeat count is not a positive number");
+                    cx.emit_err(errors::ConcatBytesBadRepeat {span: count.value.span });
                 }
             }
             &ast::ExprKind::Lit(token_lit) => match ast::LitKind::from_token_lit(token_lit) {
@@ -196,9 +175,7 @@ pub fn expand_concat_bytes(
         }
     }
     if !missing_literals.is_empty() {
-        let mut err = cx.struct_span_err(missing_literals, "expected a byte literal");
-        err.note("only byte literals (like `b\"foo\"`, `b's'`, and `[3, 4, 5]`) can be passed to `concat_bytes!()`");
-        err.emit();
+        cx.emit_err(errors::ConcatBytesMissingLiteral { spans: missing_literals });
         return base::MacEager::expr(DummyResult::raw_expr(sp, true));
     } else if has_errors {
         return base::MacEager::expr(DummyResult::raw_expr(sp, true));
diff --git a/compiler/rustc_builtin_macros/src/concat_idents.rs b/compiler/rustc_builtin_macros/src/concat_idents.rs
index 297c604e0..8c737f043 100644
--- a/compiler/rustc_builtin_macros/src/concat_idents.rs
+++ b/compiler/rustc_builtin_macros/src/concat_idents.rs
@@ -6,13 +6,15 @@ use rustc_expand::base::{self, *};
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::Span;
 
+use crate::errors;
+
 pub fn expand_concat_idents<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
     sp: Span,
     tts: TokenStream,
 ) -> Box<dyn base::MacResult + 'cx> {
     if tts.is_empty() {
-        cx.span_err(sp, "concat_idents! takes 1 or more arguments");
+        cx.emit_err(errors::ConcatIdentsMissingArgs { span: sp });
         return DummyResult::any(sp);
     }
 
@@ -22,7 +24,7 @@ pub fn expand_concat_idents<'cx>(
             match e {
                 TokenTree::Token(Token { kind: token::Comma, .. }, _) => {}
                 _ => {
-                    cx.span_err(sp, "concat_idents! expecting comma");
+                    cx.emit_err(errors::ConcatIdentsMissingComma { span: sp });
                     return DummyResult::any(sp);
                 }
             }
@@ -34,7 +36,7 @@ pub fn expand_concat_idents<'cx>(
                 }
             }
 
-            cx.span_err(sp, "concat_idents! requires ident args");
+            cx.emit_err(errors::ConcatIdentsIdentArgs { span: sp });
             return DummyResult::any(sp);
         }
     }
diff --git a/compiler/rustc_builtin_macros/src/derive.rs b/compiler/rustc_builtin_macros/src/derive.rs
index 2a8dc0284..fe4483104 100644
--- a/compiler/rustc_builtin_macros/src/derive.rs
+++ b/compiler/rustc_builtin_macros/src/derive.rs
@@ -1,8 +1,8 @@
 use crate::cfg_eval::cfg_eval;
+use crate::errors;
 
 use rustc_ast as ast;
 use rustc_ast::{GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
-use rustc_errors::{struct_span_err, Applicability};
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier};
 use rustc_feature::AttributeTemplate;
 use rustc_parse::validate_attr;
@@ -116,49 +116,33 @@ fn report_bad_target(sess: &Session, item: &Annotatable, span: Span) -> bool {
     let bad_target =
         !matches!(item_kind, Some(ItemKind::Struct(..) | ItemKind::Enum(..) | ItemKind::Union(..)));
     if bad_target {
-        struct_span_err!(
-            sess,
-            span,
-            E0774,
-            "`derive` may only be applied to `struct`s, `enum`s and `union`s",
-        )
-        .span_label(span, "not applicable here")
-        .span_label(item.span(), "not a `struct`, `enum` or `union`")
-        .emit();
+        sess.emit_err(errors::BadDeriveTarget { span, item: item.span() });
     }
     bad_target
 }
 
 fn report_unexpected_meta_item_lit(sess: &Session, lit: &ast::MetaItemLit) {
-    let help_msg = match lit.kind {
+    let help = match lit.kind {
         ast::LitKind::Str(_, ast::StrStyle::Cooked)
             if rustc_lexer::is_ident(lit.symbol.as_str()) =>
         {
-            format!("try using `#[derive({})]`", lit.symbol)
+            errors::BadDeriveLitHelp::StrLit { sym: lit.symbol }
         }
-        _ => "for example, write `#[derive(Debug)]` for `Debug`".to_string(),
+        _ => errors::BadDeriveLitHelp::Other,
     };
-    struct_span_err!(sess, lit.span, E0777, "expected path to a trait, found literal",)
-        .span_label(lit.span, "not a trait")
-        .help(&help_msg)
-        .emit();
+    sess.emit_err(errors::BadDeriveLit { span: lit.span, help });
 }
 
 fn report_path_args(sess: &Session, meta: &ast::MetaItem) {
-    let report_error = |title, action| {
-        let span = meta.span.with_lo(meta.path.span.hi());
-        sess.struct_span_err(span, title)
-            .span_suggestion(span, action, "", Applicability::MachineApplicable)
-            .emit();
-    };
+    let span = meta.span.with_lo(meta.path.span.hi());
+
     match meta.kind {
         MetaItemKind::Word => {}
-        MetaItemKind::List(..) => report_error(
-            "traits in `#[derive(...)]` don't accept arguments",
-            "remove the arguments",
-        ),
+        MetaItemKind::List(..) => {
+            sess.emit_err(errors::DerivePathArgsList { span });
+        }
         MetaItemKind::NameValue(..) => {
-            report_error("traits in `#[derive(...)]` don't accept values", "remove the value")
+            sess.emit_err(errors::DerivePathArgsValue { span });
         }
     }
 }
diff --git a/compiler/rustc_builtin_macros/src/deriving/default.rs b/compiler/rustc_builtin_macros/src/deriving/default.rs
index 4d753a2ed..33fe98b40 100644
--- a/compiler/rustc_builtin_macros/src/deriving/default.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/default.rs
@@ -1,8 +1,8 @@
 use crate::deriving::generic::ty::*;
 use crate::deriving::generic::*;
+use crate::errors;
 use rustc_ast as ast;
-use rustc_ast::{walk_list, EnumDef, VariantData};
-use rustc_errors::Applicability;
+use rustc_ast::{attr, walk_list, EnumDef, VariantData};
 use rustc_expand::base::{Annotatable, DummyResult, ExtCtxt};
 use rustc_span::symbol::Ident;
 use rustc_span::symbol::{kw, sym};
@@ -106,7 +106,7 @@ fn extract_default_variant<'a>(
     let default_variants: SmallVec<[_; 1]> = enum_def
         .variants
         .iter()
-        .filter(|variant| cx.sess.contains_name(&variant.attrs, kw::Default))
+        .filter(|variant| attr::contains_name(&variant.attrs, kw::Default))
         .collect();
 
     let variant = match default_variants.as_slice() {
@@ -116,69 +116,52 @@ fn extract_default_variant<'a>(
                 .variants
                 .iter()
                 .filter(|variant| matches!(variant.data, VariantData::Unit(..)))
-                .filter(|variant| !cx.sess.contains_name(&variant.attrs, sym::non_exhaustive));
+                .filter(|variant| !attr::contains_name(&variant.attrs, sym::non_exhaustive));
 
-            let mut diag = cx.struct_span_err(trait_span, "no default declared");
-            diag.help("make a unit variant default by placing `#[default]` above it");
-            for variant in possible_defaults {
-                // Suggest making each unit variant default.
-                diag.tool_only_span_suggestion(
-                    variant.span,
-                    &format!("make `{}` default", variant.ident),
-                    format!("#[default] {}", variant.ident),
-                    Applicability::MaybeIncorrect,
-                );
-            }
-            diag.emit();
+            let suggs = possible_defaults
+                .map(|v| errors::NoDefaultVariantSugg { span: v.span, ident: v.ident })
+                .collect();
+            cx.emit_err(errors::NoDefaultVariant { span: trait_span, suggs });
 
             return Err(());
         }
         [first, rest @ ..] => {
-            let mut diag = cx.struct_span_err(trait_span, "multiple declared defaults");
-            diag.span_label(first.span, "first default");
-            diag.span_labels(rest.iter().map(|variant| variant.span), "additional default");
-            diag.note("only one variant can be default");
-            for variant in &default_variants {
-                // Suggest making each variant already tagged default.
-                let suggestion = default_variants
-                    .iter()
-                    .filter_map(|v| {
-                        if v.span == variant.span {
-                            None
-                        } else {
-                            Some((cx.sess.find_by_name(&v.attrs, kw::Default)?.span, String::new()))
-                        }
-                    })
-                    .collect();
-
-                diag.tool_only_multipart_suggestion(
-                    &format!("make `{}` default", variant.ident),
-                    suggestion,
-                    Applicability::MaybeIncorrect,
-                );
-            }
-            diag.emit();
-
+            let suggs = default_variants
+                .iter()
+                .map(|variant| {
+                    let spans = default_variants
+                        .iter()
+                        .filter_map(|v| {
+                            if v.span == variant.span {
+                                None
+                            } else {
+                                Some(attr::find_by_name(&v.attrs, kw::Default)?.span)
+                            }
+                        })
+                        .collect();
+                    errors::MultipleDefaultsSugg { spans, ident: variant.ident }
+                })
+                .collect();
+            cx.emit_err(errors::MultipleDefaults {
+                span: trait_span,
+                first: first.span,
+                additional: rest.iter().map(|v| v.span).collect(),
+                suggs,
+            });
             return Err(());
         }
     };
 
     if !matches!(variant.data, VariantData::Unit(..)) {
-        cx.struct_span_err(
-            variant.ident.span,
-            "the `#[default]` attribute may only be used on unit enum variants",
-        )
-        .help("consider a manual implementation of `Default`")
-        .emit();
-
+        cx.emit_err(errors::NonUnitDefault { span: variant.ident.span });
         return Err(());
     }
 
-    if let Some(non_exhaustive_attr) = cx.sess.find_by_name(&variant.attrs, sym::non_exhaustive) {
-        cx.struct_span_err(variant.ident.span, "default variant must be exhaustive")
-            .span_label(non_exhaustive_attr.span, "declared `#[non_exhaustive]` here")
-            .help("consider a manual implementation of `Default`")
-            .emit();
+    if let Some(non_exhaustive_attr) = attr::find_by_name(&variant.attrs, sym::non_exhaustive) {
+        cx.emit_err(errors::NonExhaustiveDefault {
+            span: variant.ident.span,
+            non_exhaustive: non_exhaustive_attr.span,
+        });
 
         return Err(());
     }
@@ -191,7 +174,7 @@ fn validate_default_attribute(
     default_variant: &rustc_ast::Variant,
 ) -> Result<(), ()> {
     let attrs: SmallVec<[_; 1]> =
-        cx.sess.filter_by_name(&default_variant.attrs, kw::Default).collect();
+        attr::filter_by_name(&default_variant.attrs, kw::Default).collect();
 
     let attr = match attrs.as_slice() {
         [attr] => attr,
@@ -199,35 +182,23 @@ fn validate_default_attribute(
             "this method must only be called with a variant that has a `#[default]` attribute",
         ),
         [first, rest @ ..] => {
-            let suggestion_text =
-                if rest.len() == 1 { "try removing this" } else { "try removing these" };
-
-            cx.struct_span_err(default_variant.ident.span, "multiple `#[default]` attributes")
-                .note("only one `#[default]` attribute is needed")
-                .span_label(first.span, "`#[default]` used here")
-                .span_label(rest[0].span, "`#[default]` used again here")
-                .span_help(rest.iter().map(|attr| attr.span).collect::<Vec<_>>(), suggestion_text)
-                // This would otherwise display the empty replacement, hence the otherwise
-                // repetitive `.span_help` call above.
-                .tool_only_multipart_suggestion(
-                    suggestion_text,
-                    rest.iter().map(|attr| (attr.span, String::new())).collect(),
-                    Applicability::MachineApplicable,
-                )
-                .emit();
+            let sugg = errors::MultipleDefaultAttrsSugg {
+                spans: rest.iter().map(|attr| attr.span).collect(),
+            };
+            cx.emit_err(errors::MultipleDefaultAttrs {
+                span: default_variant.ident.span,
+                first: first.span,
+                first_rest: rest[0].span,
+                rest: rest.iter().map(|attr| attr.span).collect::<Vec<_>>().into(),
+                only_one: rest.len() == 1,
+                sugg,
+            });
 
             return Err(());
         }
     };
     if !attr.is_word() {
-        cx.struct_span_err(attr.span, "`#[default]` attribute does not accept a value")
-            .span_suggestion_hidden(
-                attr.span,
-                "try using `#[default]`",
-                "#[default]",
-                Applicability::MaybeIncorrect,
-            )
-            .emit();
+        cx.emit_err(errors::DefaultHasArg { span: attr.span });
 
         return Err(());
     }
@@ -241,12 +212,7 @@ struct DetectNonVariantDefaultAttr<'a, 'b> {
 impl<'a, 'b> rustc_ast::visit::Visitor<'a> for DetectNonVariantDefaultAttr<'a, 'b> {
     fn visit_attribute(&mut self, attr: &'a rustc_ast::Attribute) {
         if attr.has_name(kw::Default) {
-            self.cx
-                .struct_span_err(
-                    attr.span,
-                    "the `#[default]` attribute may only be used on unit enum variants",
-                )
-                .emit();
+            self.cx.emit_err(errors::NonUnitDefault { span: attr.span });
         }
 
         rustc_ast::visit::walk_attribute(self, attr);
diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
index 1f819beeb..e5a003315 100644
--- a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
@@ -162,7 +162,7 @@
 pub use StaticFields::*;
 pub use SubstructureFields::*;
 
-use crate::deriving;
+use crate::{deriving, errors};
 use rustc_ast::ptr::P;
 use rustc_ast::{
     self as ast, BindingAnnotation, ByRef, EnumDef, Expr, GenericArg, GenericParamKind, Generics,
@@ -415,7 +415,7 @@ fn find_type_parameters(
         }
 
         fn visit_mac_call(&mut self, mac: &ast::MacCall) {
-            self.cx.span_err(mac.span(), "`derive` cannot be used on items with type macros");
+            self.cx.emit_err(errors::DeriveMacroCall { span: mac.span() });
         }
     }
 
@@ -488,7 +488,7 @@ impl<'a> TraitDef<'a> {
                                 is_packed,
                             )
                         } else {
-                            cx.span_err(mitem.span, "this trait cannot be derived for unions");
+                            cx.emit_err(errors::DeriveUnion { span: mitem.span });
                             return;
                         }
                     }
@@ -1052,6 +1052,7 @@ impl<'a> MethodDef<'a> {
     ///         ::core::hash::Hash::hash(&{ self.y }, state)
     ///     }
     /// }
+    /// ```
     fn expand_struct_method_body<'b>(
         &self,
         cx: &mut ExtCtxt<'_>,
diff --git a/compiler/rustc_builtin_macros/src/env.rs b/compiler/rustc_builtin_macros/src/env.rs
index f011cb754..58c972738 100644
--- a/compiler/rustc_builtin_macros/src/env.rs
+++ b/compiler/rustc_builtin_macros/src/env.rs
@@ -11,6 +11,8 @@ use rustc_span::Span;
 use std::env;
 use thin_vec::thin_vec;
 
+use crate::errors;
+
 pub fn expand_option_env<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
     sp: Span,
@@ -54,7 +56,7 @@ pub fn expand_env<'cx>(
 ) -> Box<dyn base::MacResult + 'cx> {
     let mut exprs = match get_exprs_from_tts(cx, tts) {
         Some(exprs) if exprs.is_empty() || exprs.len() > 2 => {
-            cx.span_err(sp, "env! takes 1 or 2 arguments");
+            cx.emit_err(errors::EnvTakesArgs { span: sp });
             return DummyResult::any(sp);
         }
         None => return DummyResult::any(sp),
@@ -78,18 +80,12 @@ pub fn expand_env<'cx>(
     cx.sess.parse_sess.env_depinfo.borrow_mut().insert((var, value));
     let e = match value {
         None => {
-            let (msg, help) = match custom_msg {
-                None => (
-                    format!("environment variable `{var}` not defined at compile time"),
-                    Some(help_for_missing_env_var(var.as_str())),
-                ),
-                Some(s) => (s.to_string(), None),
-            };
-            let mut diag = cx.struct_span_err(sp, &msg);
-            if let Some(help) = help {
-                diag.help(help);
-            }
-            diag.emit();
+            cx.emit_err(errors::EnvNotDefined {
+                span: sp,
+                msg: custom_msg,
+                var,
+                help: custom_msg.is_none().then(|| help_for_missing_env_var(var.as_str())),
+            });
             return DummyResult::any(sp);
         }
         Some(value) => cx.expr_str(sp, value),
@@ -97,15 +93,13 @@ pub fn expand_env<'cx>(
     MacEager::expr(e)
 }
 
-fn help_for_missing_env_var(var: &str) -> String {
+fn help_for_missing_env_var(var: &str) -> errors::EnvNotDefinedHelp {
     if var.starts_with("CARGO_")
         || var.starts_with("DEP_")
         || matches!(var, "OUT_DIR" | "OPT_LEVEL" | "PROFILE" | "HOST" | "TARGET")
     {
-        format!(
-            "Cargo sets build script variables at run time. Use `std::env::var(\"{var}\")` instead"
-        )
+        errors::EnvNotDefinedHelp::CargoVar
     } else {
-        format!("Use `std::env::var(\"{var}\")` to read the variable at run time")
+        errors::EnvNotDefinedHelp::Other
     }
 }
diff --git a/compiler/rustc_builtin_macros/src/errors.rs b/compiler/rustc_builtin_macros/src/errors.rs
new file mode 100644
index 000000000..630f9b87b
--- /dev/null
+++ b/compiler/rustc_builtin_macros/src/errors.rs
@@ -0,0 +1,553 @@
+use rustc_errors::{
+    AddToDiagnostic, EmissionGuarantee, IntoDiagnostic, MultiSpan, SingleLabelManySpans,
+};
+use rustc_macros::{Diagnostic, Subdiagnostic};
+use rustc_span::{symbol::Ident, Span, Symbol};
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_requires_cfg_pattern)]
+pub(crate) struct RequiresCfgPattern {
+    #[primary_span]
+    #[label]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_expected_one_cfg_pattern)]
+pub(crate) struct OneCfgPattern {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_alloc_error_must_be_fn)]
+pub(crate) struct AllocErrorMustBeFn {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_assert_requires_boolean)]
+pub(crate) struct AssertRequiresBoolean {
+    #[primary_span]
+    #[label]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_assert_requires_expression)]
+pub(crate) struct AssertRequiresExpression {
+    #[primary_span]
+    pub(crate) span: Span,
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub(crate) token: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_assert_missing_comma)]
+pub(crate) struct AssertMissingComma {
+    #[primary_span]
+    pub(crate) span: Span,
+    #[suggestion(code = ", ", applicability = "maybe-incorrect", style = "short")]
+    pub(crate) comma: Span,
+}
+
+#[derive(Diagnostic)]
+pub(crate) enum CfgAccessibleInvalid {
+    #[diag(builtin_macros_cfg_accessible_unspecified_path)]
+    UnspecifiedPath(#[primary_span] Span),
+    #[diag(builtin_macros_cfg_accessible_multiple_paths)]
+    MultiplePaths(#[primary_span] Span),
+    #[diag(builtin_macros_cfg_accessible_literal_path)]
+    LiteralPath(#[primary_span] Span),
+    #[diag(builtin_macros_cfg_accessible_has_args)]
+    HasArguments(#[primary_span] Span),
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_cfg_accessible_indeterminate)]
+pub(crate) struct CfgAccessibleIndeterminate {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_missing_literal)]
+#[note]
+pub(crate) struct ConcatMissingLiteral {
+    #[primary_span]
+    pub(crate) spans: Vec<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_bytestr)]
+pub(crate) struct ConcatBytestr {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_bytes_invalid)]
+pub(crate) struct ConcatBytesInvalid {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) lit_kind: &'static str,
+    #[subdiagnostic]
+    pub(crate) sugg: Option<ConcatBytesInvalidSuggestion>,
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum ConcatBytesInvalidSuggestion {
+    #[suggestion(
+        builtin_macros_byte_char,
+        code = "b{snippet}",
+        applicability = "machine-applicable"
+    )]
+    CharLit {
+        #[primary_span]
+        span: Span,
+        snippet: String,
+    },
+    #[suggestion(
+        builtin_macros_byte_str,
+        code = "b{snippet}",
+        applicability = "machine-applicable"
+    )]
+    StrLit {
+        #[primary_span]
+        span: Span,
+        snippet: String,
+    },
+    #[suggestion(
+        builtin_macros_number_array,
+        code = "[{snippet}]",
+        applicability = "machine-applicable"
+    )]
+    IntLit {
+        #[primary_span]
+        span: Span,
+        snippet: String,
+    },
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_bytes_oob)]
+pub(crate) struct ConcatBytesOob {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_bytes_non_u8)]
+pub(crate) struct ConcatBytesNonU8 {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_bytes_missing_literal)]
+#[note]
+pub(crate) struct ConcatBytesMissingLiteral {
+    #[primary_span]
+    pub(crate) spans: Vec<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_bytes_array)]
+pub(crate) struct ConcatBytesArray {
+    #[primary_span]
+    pub(crate) span: Span,
+    #[note]
+    #[help]
+    pub(crate) bytestr: bool,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_bytes_bad_repeat)]
+pub(crate) struct ConcatBytesBadRepeat {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_idents_missing_args)]
+pub(crate) struct ConcatIdentsMissingArgs {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_idents_missing_comma)]
+pub(crate) struct ConcatIdentsMissingComma {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_concat_idents_ident_args)]
+pub(crate) struct ConcatIdentsIdentArgs {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_bad_derive_target, code = "E0774")]
+pub(crate) struct BadDeriveTarget {
+    #[primary_span]
+    #[label]
+    pub(crate) span: Span,
+    #[label(builtin_macros_label2)]
+    pub(crate) item: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_unexpected_lit, code = "E0777")]
+pub(crate) struct BadDeriveLit {
+    #[primary_span]
+    #[label]
+    pub(crate) span: Span,
+    #[subdiagnostic]
+    pub help: BadDeriveLitHelp,
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum BadDeriveLitHelp {
+    #[help(builtin_macros_str_lit)]
+    StrLit { sym: Symbol },
+    #[help(builtin_macros_other)]
+    Other,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_derive_path_args_list)]
+pub(crate) struct DerivePathArgsList {
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_derive_path_args_value)]
+pub(crate) struct DerivePathArgsValue {
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_no_default_variant)]
+#[help]
+pub(crate) struct NoDefaultVariant {
+    #[primary_span]
+    pub(crate) span: Span,
+    #[subdiagnostic]
+    pub(crate) suggs: Vec<NoDefaultVariantSugg>,
+}
+
+#[derive(Subdiagnostic)]
+#[suggestion(
+    builtin_macros_suggestion,
+    code = "#[default] {ident}",
+    applicability = "maybe-incorrect",
+    style = "tool-only"
+)]
+pub(crate) struct NoDefaultVariantSugg {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) ident: Ident,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_multiple_defaults)]
+#[note]
+pub(crate) struct MultipleDefaults {
+    #[primary_span]
+    pub(crate) span: Span,
+    #[label]
+    pub(crate) first: Span,
+    #[label(builtin_macros_additional)]
+    pub additional: Vec<Span>,
+    #[subdiagnostic]
+    pub suggs: Vec<MultipleDefaultsSugg>,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    builtin_macros_suggestion,
+    applicability = "maybe-incorrect",
+    style = "tool-only"
+)]
+pub(crate) struct MultipleDefaultsSugg {
+    #[suggestion_part(code = "")]
+    pub(crate) spans: Vec<Span>,
+    pub(crate) ident: Ident,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_non_unit_default)]
+#[help]
+pub(crate) struct NonUnitDefault {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_non_exhaustive_default)]
+#[help]
+pub(crate) struct NonExhaustiveDefault {
+    #[primary_span]
+    pub(crate) span: Span,
+    #[label]
+    pub(crate) non_exhaustive: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_multiple_default_attrs)]
+#[note]
+pub(crate) struct MultipleDefaultAttrs {
+    #[primary_span]
+    pub(crate) span: Span,
+    #[label]
+    pub(crate) first: Span,
+    #[label(builtin_macros_label_again)]
+    pub(crate) first_rest: Span,
+    #[help]
+    pub(crate) rest: MultiSpan,
+    pub(crate) only_one: bool,
+    #[subdiagnostic]
+    pub(crate) sugg: MultipleDefaultAttrsSugg,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    builtin_macros_help,
+    applicability = "machine-applicable",
+    style = "tool-only"
+)]
+pub(crate) struct MultipleDefaultAttrsSugg {
+    #[suggestion_part(code = "")]
+    pub(crate) spans: Vec<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_default_arg)]
+pub(crate) struct DefaultHasArg {
+    #[primary_span]
+    #[suggestion(code = "#[default]", style = "hidden", applicability = "maybe-incorrect")]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_derive_macro_call)]
+pub(crate) struct DeriveMacroCall {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_cannot_derive_union)]
+pub(crate) struct DeriveUnion {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_env_takes_args)]
+pub(crate) struct EnvTakesArgs {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+//#[derive(Diagnostic)]
+//#[diag(builtin_macros_env_not_defined)]
+pub(crate) struct EnvNotDefined {
+    pub(crate) span: Span,
+    pub(crate) msg: Option<Symbol>,
+    pub(crate) var: Symbol,
+    pub(crate) help: Option<EnvNotDefinedHelp>,
+}
+
+// Hand-written implementation to support custom user messages
+impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for EnvNotDefined {
+    #[track_caller]
+    fn into_diagnostic(
+        self,
+        handler: &'a rustc_errors::Handler,
+    ) -> rustc_errors::DiagnosticBuilder<'a, G> {
+        let mut diag = if let Some(msg) = self.msg {
+            handler.struct_diagnostic(msg.as_str())
+        } else {
+            handler.struct_diagnostic(crate::fluent_generated::builtin_macros_env_not_defined)
+        };
+        diag.set_arg("var", self.var);
+        diag.set_span(self.span);
+        if let Some(help) = self.help {
+            diag.subdiagnostic(help);
+        }
+        diag
+    }
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum EnvNotDefinedHelp {
+    #[help(builtin_macros_cargo)]
+    CargoVar,
+    #[help(builtin_macros_other)]
+    Other,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_format_requires_string)]
+pub(crate) struct FormatRequiresString {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_format_duplicate_arg)]
+pub(crate) struct FormatDuplicateArg {
+    #[primary_span]
+    pub(crate) span: Span,
+    #[label(builtin_macros_label1)]
+    pub(crate) prev: Span,
+    #[label(builtin_macros_label2)]
+    pub(crate) duplicate: Span,
+    pub(crate) ident: Ident,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_format_positional_after_named)]
+pub(crate) struct PositionalAfterNamed {
+    #[primary_span]
+    #[label]
+    pub(crate) span: Span,
+    #[label(builtin_macros_named_args)]
+    pub(crate) args: Vec<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_format_string_invalid)]
+pub(crate) struct InvalidFormatString {
+    #[primary_span]
+    #[label]
+    pub(crate) span: Span,
+    pub(crate) desc: String,
+    pub(crate) label1: String,
+    #[subdiagnostic]
+    pub(crate) note_: Option<InvalidFormatStringNote>,
+    #[subdiagnostic]
+    pub(crate) label_: Option<InvalidFormatStringLabel>,
+    #[subdiagnostic]
+    pub(crate) sugg_: Option<InvalidFormatStringSuggestion>,
+}
+
+#[derive(Subdiagnostic)]
+#[note(builtin_macros_note)]
+pub(crate) struct InvalidFormatStringNote {
+    pub(crate) note: String,
+}
+
+#[derive(Subdiagnostic)]
+#[label(builtin_macros_second_label)]
+pub(crate) struct InvalidFormatStringLabel {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) label: String,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    builtin_macros_sugg,
+    style = "verbose",
+    applicability = "machine-applicable"
+)]
+pub(crate) struct InvalidFormatStringSuggestion {
+    #[suggestion_part(code = "{len}")]
+    pub(crate) captured: Span,
+    pub(crate) len: String,
+    #[suggestion_part(code = ", {arg}")]
+    pub(crate) span: Span,
+    pub(crate) arg: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_format_no_arg_named)]
+#[note]
+#[note(builtin_macros_note2)]
+pub(crate) struct FormatNoArgNamed {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) name: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_format_unknown_trait)]
+#[note]
+pub(crate) struct FormatUnknownTrait<'a> {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) ty: &'a str,
+    #[subdiagnostic]
+    pub(crate) suggs: Vec<FormatUnknownTraitSugg>,
+}
+
+#[derive(Subdiagnostic)]
+#[suggestion(
+    builtin_macros_suggestion,
+    code = "{fmt}",
+    style = "tool-only",
+    applicability = "maybe-incorrect"
+)]
+pub struct FormatUnknownTraitSugg {
+    #[primary_span]
+    pub span: Span,
+    pub fmt: &'static str,
+    pub trait_name: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_format_unused_arg)]
+pub(crate) struct FormatUnusedArg {
+    #[primary_span]
+    #[label(builtin_macros_format_unused_arg)]
+    pub(crate) span: Span,
+    pub(crate) named: bool,
+}
+
+// Allow the singular form to be a subdiagnostic of the multiple-unused
+// form of diagnostic.
+impl AddToDiagnostic for FormatUnusedArg {
+    fn add_to_diagnostic_with<F>(self, diag: &mut rustc_errors::Diagnostic, f: F)
+    where
+        F: Fn(
+            &mut rustc_errors::Diagnostic,
+            rustc_errors::SubdiagnosticMessage,
+        ) -> rustc_errors::SubdiagnosticMessage,
+    {
+        diag.set_arg("named", self.named);
+        let msg = f(diag, crate::fluent_generated::builtin_macros_format_unused_arg.into());
+        diag.span_label(self.span, msg);
+    }
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_format_unused_args)]
+pub(crate) struct FormatUnusedArgs {
+    #[primary_span]
+    pub(crate) unused: Vec<Span>,
+    #[label]
+    pub(crate) fmt: Span,
+    #[subdiagnostic]
+    pub(crate) unused_labels: Vec<FormatUnusedArg>,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_format_pos_mismatch)]
+pub(crate) struct FormatPositionalMismatch {
+    #[primary_span]
+    pub(crate) span: MultiSpan,
+    pub(crate) n: usize,
+    pub(crate) desc: String,
+    #[subdiagnostic]
+    pub(crate) highlight: SingleLabelManySpans,
+}
diff --git a/compiler/rustc_builtin_macros/src/format.rs b/compiler/rustc_builtin_macros/src/format.rs
index e93a23394..f0fc61d7c 100644
--- a/compiler/rustc_builtin_macros/src/format.rs
+++ b/compiler/rustc_builtin_macros/src/format.rs
@@ -7,7 +7,7 @@ use rustc_ast::{
     FormatDebugHex, FormatOptions, FormatPlaceholder, FormatSign, FormatTrait,
 };
 use rustc_data_structures::fx::FxHashSet;
-use rustc_errors::{pluralize, Applicability, MultiSpan, PResult};
+use rustc_errors::{Applicability, MultiSpan, PResult, SingleLabelManySpans};
 use rustc_expand::base::{self, *};
 use rustc_parse_format as parse;
 use rustc_span::symbol::{Ident, Symbol};
@@ -36,6 +36,23 @@ enum PositionUsedAs {
 }
 use PositionUsedAs::*;
 
+use crate::errors;
+
+struct MacroInput {
+    fmtstr: P<Expr>,
+    args: FormatArguments,
+    /// Whether the first argument was a string literal or a result from eager macro expansion.
+    /// If it's not a string literal, we disallow implicit argument capturing.
+    ///
+    /// This does not correspond to whether we can treat spans to the literal normally, as the whole
+    /// invocation might be the result of another macro expansion, in which case this flag may still be true.
+    ///
+    /// See [RFC 2795] for more information.
+    ///
+    /// [RFC 2795]: https://rust-lang.github.io/rfcs/2795-format-args-implicit-identifiers.html#macro-hygiene
+    is_direct_literal: bool,
+}
+
 /// Parses the arguments from the given list of tokens, returning the diagnostic
 /// if there's a parse error so we can continue parsing other format!
 /// expressions.
@@ -45,39 +62,31 @@ use PositionUsedAs::*;
 /// ```text
 /// Ok((fmtstr, parsed arguments))
 /// ```
-fn parse_args<'a>(
-    ecx: &mut ExtCtxt<'a>,
-    sp: Span,
-    tts: TokenStream,
-) -> PResult<'a, (P<Expr>, FormatArguments)> {
+fn parse_args<'a>(ecx: &mut ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, MacroInput> {
     let mut args = FormatArguments::new();
 
     let mut p = ecx.new_parser_from_tts(tts);
 
     if p.token == token::Eof {
-        return Err(ecx.struct_span_err(sp, "requires at least a format string argument"));
+        return Err(ecx.create_err(errors::FormatRequiresString { span: sp }));
     }
 
     let first_token = &p.token;
-    let fmtstr = match first_token.kind {
-        token::TokenKind::Literal(token::Lit {
-            kind: token::LitKind::Str | token::LitKind::StrRaw(_),
-            ..
-        }) => {
-            // If the first token is a string literal, then a format expression
-            // is constructed from it.
-            //
-            // This allows us to properly handle cases when the first comma
-            // after the format string is mistakenly replaced with any operator,
-            // which cause the expression parser to eat too much tokens.
-            p.parse_literal_maybe_minus()?
-        }
-        _ => {
-            // Otherwise, we fall back to the expression parser.
-            p.parse_expr()?
-        }
+
+    let fmtstr = if let token::Literal(lit) = first_token.kind && matches!(lit.kind, token::Str | token::StrRaw(_)) {
+        // This allows us to properly handle cases when the first comma
+        // after the format string is mistakenly replaced with any operator,
+        // which cause the expression parser to eat too much tokens.
+        p.parse_literal_maybe_minus()?
+    } else {
+        // Otherwise, we fall back to the expression parser.
+        p.parse_expr()?
     };
 
+    // Only allow implicit captures to be used when the argument is a direct literal
+    // instead of a macro expanding to one.
+    let is_direct_literal = matches!(fmtstr.kind, ExprKind::Lit(_));
+
     let mut first = true;
 
     while p.token != token::Eof {
@@ -114,13 +123,12 @@ fn parse_args<'a>(
                 p.expect(&token::Eq)?;
                 let expr = p.parse_expr()?;
                 if let Some((_, prev)) = args.by_name(ident.name) {
-                    ecx.struct_span_err(
-                        ident.span,
-                        &format!("duplicate argument named `{}`", ident),
-                    )
-                    .span_label(prev.kind.ident().unwrap().span, "previously here")
-                    .span_label(ident.span, "duplicate argument")
-                    .emit();
+                    ecx.emit_err(errors::FormatDuplicateArg {
+                        span: ident.span,
+                        prev: prev.kind.ident().unwrap().span,
+                        duplicate: ident.span,
+                        ident,
+                    });
                     continue;
                 }
                 args.add(FormatArgument { kind: FormatArgumentKind::Named(ident), expr });
@@ -128,36 +136,39 @@ fn parse_args<'a>(
             _ => {
                 let expr = p.parse_expr()?;
                 if !args.named_args().is_empty() {
-                    let mut err = ecx.struct_span_err(
-                        expr.span,
-                        "positional arguments cannot follow named arguments",
-                    );
-                    err.span_label(
-                        expr.span,
-                        "positional arguments must be before named arguments",
-                    );
-                    for arg in args.named_args() {
-                        if let Some(name) = arg.kind.ident() {
-                            err.span_label(name.span.to(arg.expr.span), "named argument");
-                        }
-                    }
-                    err.emit();
+                    ecx.emit_err(errors::PositionalAfterNamed {
+                        span: expr.span,
+                        args: args
+                            .named_args()
+                            .iter()
+                            .filter_map(|a| {
+                                if let Some(ident) = a.kind.ident() {
+                                    Some((a, ident))
+                                } else {
+                                    None
+                                }
+                            })
+                            .map(|(arg, n)| n.span.to(arg.expr.span))
+                            .collect(),
+                    });
                 }
                 args.add(FormatArgument { kind: FormatArgumentKind::Normal, expr });
             }
         }
     }
-    Ok((fmtstr, args))
+    Ok(MacroInput { fmtstr, args, is_direct_literal })
 }
 
-pub fn make_format_args(
+fn make_format_args(
     ecx: &mut ExtCtxt<'_>,
-    efmt: P<Expr>,
-    mut args: FormatArguments,
+    input: MacroInput,
     append_newline: bool,
 ) -> Result<FormatArgs, ()> {
     let msg = "format argument must be a string literal";
-    let unexpanded_fmt_span = efmt.span;
+    let unexpanded_fmt_span = input.fmtstr.span;
+
+    let MacroInput { fmtstr: efmt, mut args, is_direct_literal } = input;
+
     let (fmt_str, fmt_style, fmt_span) = match expr_to_spanned_string(ecx, efmt, msg) {
         Ok(mut fmt) if append_newline => {
             fmt.0 = Symbol::intern(&format!("{}\n", fmt.0));
@@ -208,11 +219,11 @@ pub fn make_format_args(
         }
     }
 
-    let is_literal = parser.is_literal;
+    let is_source_literal = parser.is_source_literal;
 
     if !parser.errors.is_empty() {
         let err = parser.errors.remove(0);
-        let sp = if is_literal {
+        let sp = if is_source_literal {
             fmt_span.from_inner(InnerSpan::new(err.span.start, err.span.end))
         } else {
             // The format string could be another macro invocation, e.g.:
@@ -225,13 +236,19 @@ pub fn make_format_args(
             // argument span here.
             fmt_span
         };
-        let mut e = ecx.struct_span_err(sp, &format!("invalid format string: {}", err.description));
-        e.span_label(sp, err.label + " in format string");
+        let mut e = errors::InvalidFormatString {
+            span: sp,
+            note_: None,
+            label_: None,
+            sugg_: None,
+            desc: err.description,
+            label1: err.label,
+        };
         if let Some(note) = err.note {
-            e.note(&note);
+            e.note_ = Some(errors::InvalidFormatStringNote { note });
         }
-        if let Some((label, span)) = err.secondary_label && is_literal {
-            e.span_label(fmt_span.from_inner(InnerSpan::new(span.start, span.end)), label);
+        if let Some((label, span)) = err.secondary_label && is_source_literal {
+            e.label_ = Some(errors::InvalidFormatStringLabel { span: fmt_span.from_inner(InnerSpan::new(span.start, span.end)), label } );
         }
         if err.should_be_replaced_with_positional_argument {
             let captured_arg_span =
@@ -241,22 +258,20 @@ pub fn make_format_args(
                     Some(arg) => arg.expr.span,
                     None => fmt_span,
                 };
-                e.multipart_suggestion_verbose(
-                    "consider using a positional formatting argument instead",
-                    vec![
-                        (captured_arg_span, args.unnamed_args().len().to_string()),
-                        (span.shrink_to_hi(), format!(", {}", arg)),
-                    ],
-                    Applicability::MachineApplicable,
-                );
+                e.sugg_ = Some(errors::InvalidFormatStringSuggestion {
+                    captured: captured_arg_span,
+                    len: args.unnamed_args().len().to_string(),
+                    span: span.shrink_to_hi(),
+                    arg,
+                });
             }
         }
-        e.emit();
+        ecx.emit_err(e);
         return Err(());
     }
 
     let to_span = |inner_span: rustc_parse_format::InnerSpan| {
-        is_literal.then(|| {
+        is_source_literal.then(|| {
             fmt_span.from_inner(InnerSpan { start: inner_span.start, end: inner_span.end })
         })
     };
@@ -304,15 +319,12 @@ pub fn make_format_args(
                     // Name not found in `args`, so we add it as an implicitly captured argument.
                     let span = span.unwrap_or(fmt_span);
                     let ident = Ident::new(name, span);
-                    let expr = if is_literal {
+                    let expr = if is_direct_literal {
                         ecx.expr_ident(span, ident)
                     } else {
                         // For the moment capturing variables from format strings expanded from macros is
                         // disabled (see RFC #2795)
-                        ecx.struct_span_err(span, &format!("there is no argument named `{name}`"))
-                            .note(format!("did you intend to capture a variable `{name}` from the surrounding scope?"))
-                            .note("to avoid ambiguity, `format_args!` cannot capture variables when the format string is expanded from a macro")
-                            .emit();
+                        ecx.emit_err(errors::FormatNoArgNamed { span, name });
                         DummyResult::raw_expr(span, true)
                     };
                     Ok(args.add(FormatArgument { kind: FormatArgumentKind::Captured(ident), expr }))
@@ -466,12 +478,8 @@ pub fn make_format_args(
         .enumerate()
         .filter(|&(_, used)| !used)
         .map(|(i, _)| {
-            let msg = if let FormatArgumentKind::Named(_) = args.explicit_args()[i].kind {
-                "named argument never used"
-            } else {
-                "argument never used"
-            };
-            (args.explicit_args()[i].expr.span, msg)
+            let named = matches!(args.explicit_args()[i].kind, FormatArgumentKind::Named(_));
+            (args.explicit_args()[i].expr.span, named)
         })
         .collect::<Vec<_>>();
 
@@ -522,22 +530,8 @@ fn invalid_placeholder_type_error(
     fmt_span: Span,
 ) {
     let sp = ty_span.map(|sp| fmt_span.from_inner(InnerSpan::new(sp.start, sp.end)));
-    let mut err =
-        ecx.struct_span_err(sp.unwrap_or(fmt_span), &format!("unknown format trait `{}`", ty));
-    err.note(
-        "the only appropriate formatting traits are:\n\
-                                - ``, which uses the `Display` trait\n\
-                                - `?`, which uses the `Debug` trait\n\
-                                - `e`, which uses the `LowerExp` trait\n\
-                                - `E`, which uses the `UpperExp` trait\n\
-                                - `o`, which uses the `Octal` trait\n\
-                                - `p`, which uses the `Pointer` trait\n\
-                                - `b`, which uses the `Binary` trait\n\
-                                - `x`, which uses the `LowerHex` trait\n\
-                                - `X`, which uses the `UpperHex` trait",
-    );
-    if let Some(sp) = sp {
-        for (fmt, name) in &[
+    let suggs = if let Some(sp) = sp {
+        [
             ("", "Display"),
             ("?", "Debug"),
             ("e", "LowerExp"),
@@ -547,40 +541,38 @@ fn invalid_placeholder_type_error(
             ("b", "Binary"),
             ("x", "LowerHex"),
             ("X", "UpperHex"),
-        ] {
-            err.tool_only_span_suggestion(
-                sp,
-                &format!("use the `{}` trait", name),
-                *fmt,
-                Applicability::MaybeIncorrect,
-            );
-        }
-    }
-    err.emit();
+        ]
+        .into_iter()
+        .map(|(fmt, trait_name)| errors::FormatUnknownTraitSugg { span: sp, fmt, trait_name })
+        .collect()
+    } else {
+        vec![]
+    };
+    ecx.emit_err(errors::FormatUnknownTrait { span: sp.unwrap_or(fmt_span), ty, suggs });
 }
 
 fn report_missing_placeholders(
     ecx: &mut ExtCtxt<'_>,
-    unused: Vec<(Span, &str)>,
+    unused: Vec<(Span, bool)>,
     detect_foreign_fmt: bool,
     str_style: Option<usize>,
     fmt_str: &str,
     fmt_span: Span,
 ) {
-    let mut diag = if let &[(span, msg)] = &unused[..] {
-        let mut diag = ecx.struct_span_err(span, msg);
-        diag.span_label(span, msg);
-        diag
+    let mut diag = if let &[(span, named)] = &unused[..] {
+        //let mut diag = ecx.struct_span_err(span, msg);
+        //diag.span_label(span, msg);
+        //diag
+        ecx.create_err(errors::FormatUnusedArg { span, named })
     } else {
-        let mut diag = ecx.struct_span_err(
-            unused.iter().map(|&(sp, _)| sp).collect::<Vec<Span>>(),
-            "multiple unused formatting arguments",
-        );
-        diag.span_label(fmt_span, "multiple missing formatting specifiers");
-        for &(span, msg) in &unused {
-            diag.span_label(span, msg);
-        }
-        diag
+        let unused_labels =
+            unused.iter().map(|&(span, named)| errors::FormatUnusedArg { span, named }).collect();
+        let unused_spans = unused.iter().map(|&(span, _)| span).collect();
+        ecx.create_err(errors::FormatUnusedArgs {
+            fmt: fmt_span,
+            unused: unused_spans,
+            unused_labels,
+        })
     };
 
     // Used to ensure we only report translations for *one* kind of foreign format.
@@ -759,18 +751,16 @@ fn report_invalid_references(
         } else {
             MultiSpan::from_spans(spans)
         };
-        e = ecx.struct_span_err(
+        e = ecx.create_err(errors::FormatPositionalMismatch {
             span,
-            &format!(
-                "{} positional argument{} in format string, but {}",
-                num_placeholders,
-                pluralize!(num_placeholders),
-                num_args_desc,
-            ),
-        );
-        for arg in args.explicit_args() {
-            e.span_label(arg.expr.span, "");
-        }
+            n: num_placeholders,
+            desc: num_args_desc,
+            highlight: SingleLabelManySpans {
+                spans: args.explicit_args().iter().map(|arg| arg.expr.span).collect(),
+                label: "",
+                kind: rustc_errors::LabelKind::Label,
+            },
+        });
         // Point out `{:.*}` placeholders: those take an extra argument.
         let mut has_precision_star = false;
         for piece in template {
@@ -814,7 +804,7 @@ fn report_invalid_references(
         // for `println!("{7:7$}", 1);`
         indexes.sort();
         indexes.dedup();
-        let span: MultiSpan = if !parser.is_literal || parser.arg_places.is_empty() {
+        let span: MultiSpan = if !parser.is_source_literal || parser.arg_places.is_empty() {
             MultiSpan::from_span(fmt_span)
         } else {
             MultiSpan::from_spans(invalid_refs.iter().filter_map(|&(_, span, _, _)| span).collect())
@@ -855,8 +845,8 @@ fn expand_format_args_impl<'cx>(
 ) -> Box<dyn base::MacResult + 'cx> {
     sp = ecx.with_def_site_ctxt(sp);
     match parse_args(ecx, sp, tts) {
-        Ok((efmt, args)) => {
-            if let Ok(format_args) = make_format_args(ecx, efmt, args, nl) {
+        Ok(input) => {
+            if let Ok(format_args) = make_format_args(ecx, input, nl) {
                 MacEager::expr(ecx.expr(sp, ExprKind::FormatArgs(P(format_args))))
             } else {
                 MacEager::expr(DummyResult::raw_expr(sp, true))
diff --git a/compiler/rustc_builtin_macros/src/global_allocator.rs b/compiler/rustc_builtin_macros/src/global_allocator.rs
index 41b51bae7..866cc5adb 100644
--- a/compiler/rustc_builtin_macros/src/global_allocator.rs
+++ b/compiler/rustc_builtin_macros/src/global_allocator.rs
@@ -25,12 +25,12 @@ pub fn expand(
     // FIXME - if we get deref patterns, use them to reduce duplication here
     let (item, is_stmt, ty_span) =
         if let Annotatable::Item(item) = &item
-            && let ItemKind::Static(ty, ..) = &item.kind
+            && let ItemKind::Static(box ast::StaticItem { ty, ..}) = &item.kind
         {
             (item, false, ecx.with_def_site_ctxt(ty.span))
         } else if let Annotatable::Stmt(stmt) = &item
             && let StmtKind::Item(item) = &stmt.kind
-            && let ItemKind::Static(ty, ..) = &item.kind
+            && let ItemKind::Static(box ast::StaticItem { ty, ..}) = &item.kind
         {
             (item, true, ecx.with_def_site_ctxt(ty.span))
         } else {
diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs
index 8afb6e560..37fbd03a6 100644
--- a/compiler/rustc_builtin_macros/src/lib.rs
+++ b/compiler/rustc_builtin_macros/src/lib.rs
@@ -7,9 +7,9 @@
 #![feature(box_patterns)]
 #![feature(decl_macro)]
 #![feature(if_let_guard)]
-#![feature(is_some_and)]
 #![feature(is_sorted)]
 #![feature(let_chains)]
+#![feature(lint_reasons)]
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_quote)]
 #![recursion_limit = "256"]
@@ -40,6 +40,7 @@ mod derive;
 mod deriving;
 mod edition_panic;
 mod env;
+mod errors;
 mod format;
 mod format_foreign;
 mod global_allocator;
@@ -56,7 +57,7 @@ pub mod proc_macro_harness;
 pub mod standard_library_imports;
 pub mod test_harness;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
     let mut register = |name, kind| resolver.register_builtin_macro(name, kind);
diff --git a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
index bc513607d..378d5f39f 100644
--- a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
+++ b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
@@ -1,6 +1,6 @@
 use rustc_ast::ptr::P;
 use rustc_ast::visit::{self, Visitor};
-use rustc_ast::{self as ast, NodeId};
+use rustc_ast::{self as ast, attr, NodeId};
 use rustc_ast_pretty::pprust;
 use rustc_expand::base::{parse_macro_name_and_helper_attrs, ExtCtxt, ResolverExpand};
 use rustc_expand::expand::{AstFragment, ExpansionConfig};
@@ -34,7 +34,6 @@ enum ProcMacro {
 }
 
 struct CollectProcMacros<'a> {
-    sess: &'a Session,
     macros: Vec<ProcMacro>,
     in_root: bool,
     handler: &'a rustc_errors::Handler,
@@ -44,19 +43,18 @@ struct CollectProcMacros<'a> {
 }
 
 pub fn inject(
+    krate: &mut ast::Crate,
     sess: &Session,
     resolver: &mut dyn ResolverExpand,
-    mut krate: ast::Crate,
     is_proc_macro_crate: bool,
     has_proc_macro_decls: bool,
     is_test_crate: bool,
     handler: &rustc_errors::Handler,
-) -> ast::Crate {
+) {
     let ecfg = ExpansionConfig::default("proc_macro".to_string());
     let mut cx = ExtCtxt::new(sess, ecfg, resolver, None);
 
     let mut collect = CollectProcMacros {
-        sess,
         macros: Vec::new(),
         in_root: true,
         handler,
@@ -66,22 +64,20 @@ pub fn inject(
     };
 
     if has_proc_macro_decls || is_proc_macro_crate {
-        visit::walk_crate(&mut collect, &krate);
+        visit::walk_crate(&mut collect, krate);
     }
     let macros = collect.macros;
 
     if !is_proc_macro_crate {
-        return krate;
+        return;
     }
 
     if is_test_crate {
-        return krate;
+        return;
     }
 
     let decls = mk_decls(&mut cx, &macros);
     krate.items.push(decls);
-
-    krate
 }
 
 impl<'a> CollectProcMacros<'a> {
@@ -160,7 +156,7 @@ impl<'a> CollectProcMacros<'a> {
 impl<'a> Visitor<'a> for CollectProcMacros<'a> {
     fn visit_item(&mut self, item: &'a ast::Item) {
         if let ast::ItemKind::MacroDef(..) = item.kind {
-            if self.is_proc_macro_crate && self.sess.contains_name(&item.attrs, sym::macro_export) {
+            if self.is_proc_macro_crate && attr::contains_name(&item.attrs, sym::macro_export) {
                 let msg =
                     "cannot export macro_rules! macros from a `proc-macro` crate type currently";
                 self.handler.span_err(self.source_map.guess_head_span(item.span), msg);
@@ -176,7 +172,7 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> {
         let mut found_attr: Option<&'a ast::Attribute> = None;
 
         for attr in &item.attrs {
-            if self.sess.is_proc_macro_attr(&attr) {
+            if attr.is_proc_macro_attr() {
                 if let Some(prev_attr) = found_attr {
                     let prev_item = prev_attr.get_normal_item();
                     let item = attr.get_normal_item();
diff --git a/compiler/rustc_builtin_macros/src/standard_library_imports.rs b/compiler/rustc_builtin_macros/src/standard_library_imports.rs
index e67c0dba6..6493c6f13 100644
--- a/compiler/rustc_builtin_macros/src/standard_library_imports.rs
+++ b/compiler/rustc_builtin_macros/src/standard_library_imports.rs
@@ -1,4 +1,4 @@
-use rustc_ast as ast;
+use rustc_ast::{self as ast, attr};
 use rustc_expand::base::{ExtCtxt, ResolverExpand};
 use rustc_expand::expand::ExpansionConfig;
 use rustc_session::Session;
@@ -9,17 +9,19 @@ use rustc_span::DUMMY_SP;
 use thin_vec::thin_vec;
 
 pub fn inject(
-    mut krate: ast::Crate,
+    krate: &mut ast::Crate,
+    pre_configured_attrs: &[ast::Attribute],
     resolver: &mut dyn ResolverExpand,
     sess: &Session,
-) -> ast::Crate {
+) -> usize {
+    let orig_num_items = krate.items.len();
     let edition = sess.parse_sess.edition;
 
     // the first name in this list is the crate name of the crate with the prelude
-    let names: &[Symbol] = if sess.contains_name(&krate.attrs, sym::no_core) {
-        return krate;
-    } else if sess.contains_name(&krate.attrs, sym::no_std) {
-        if sess.contains_name(&krate.attrs, sym::compiler_builtins) {
+    let names: &[Symbol] = if attr::contains_name(pre_configured_attrs, sym::no_core) {
+        return 0;
+    } else if attr::contains_name(pre_configured_attrs, sym::no_std) {
+        if attr::contains_name(pre_configured_attrs, sym::compiler_builtins) {
             &[sym::core]
         } else {
             &[sym::core, sym::compiler_builtins]
@@ -88,6 +90,5 @@ pub fn inject(
     );
 
     krate.items.insert(0, use_item);
-
-    krate
+    krate.items.len() - orig_num_items
 }
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index e02c7e6c0..79d8be248 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -1,14 +1,13 @@
 /// The expansion from a test function to the appropriate test struct for libtest
 /// Ideally, this code would be in libtest but for efficiency and error messages it lives here.
 use crate::util::{check_builtin_macro_attribute, warn_on_duplicate_attribute};
-use rustc_ast as ast;
 use rustc_ast::ptr::P;
+use rustc_ast::{self as ast, attr};
 use rustc_ast_pretty::pprust;
 use rustc_errors::Applicability;
 use rustc_expand::base::*;
-use rustc_session::Session;
 use rustc_span::symbol::{sym, Ident, Symbol};
-use rustc_span::Span;
+use rustc_span::{FileNameDisplayPreference, Span};
 use std::iter;
 use thin_vec::{thin_vec, ThinVec};
 
@@ -33,7 +32,23 @@ pub fn expand_test_case(
     }
 
     let sp = ecx.with_def_site_ctxt(attr_sp);
-    let mut item = anno_item.expect_item();
+    let (mut item, is_stmt) = match anno_item {
+        Annotatable::Item(item) => (item, false),
+        Annotatable::Stmt(stmt) if let ast::StmtKind::Item(_) = stmt.kind => if let ast::StmtKind::Item(i) = stmt.into_inner().kind {
+            (i, true)
+        } else {
+            unreachable!()
+        },
+        _ => {
+            ecx.struct_span_err(
+                anno_item.span(),
+                "`#[test_case]` attribute is only allowed on items",
+            )
+            .emit();
+
+            return vec![];
+        }
+    };
     item = item.map(|mut item| {
         let test_path_symbol = Symbol::intern(&item_path(
             // skip the name of the root module
@@ -50,7 +65,13 @@ pub fn expand_test_case(
         item
     });
 
-    return vec![Annotatable::Item(item)];
+    let ret = if is_stmt {
+        Annotatable::Stmt(P(ecx.stmt_item(item.span, item)))
+    } else {
+        Annotatable::Item(item)
+    };
+
+    vec![ret]
 }
 
 pub fn expand_test(
@@ -97,34 +118,22 @@ pub fn expand_test_or_bench(
             }
         }
         other => {
-            cx.struct_span_err(
-                other.span(),
-                "`#[test]` attribute is only allowed on non associated functions",
-            )
-            .emit();
+            not_testable_error(cx, attr_sp, None);
             return vec![other];
         }
     };
 
-    // Note: non-associated fn items are already handled by `expand_test_or_bench`
     let ast::ItemKind::Fn(fn_) = &item.kind else {
-        let diag = &cx.sess.parse_sess.span_diagnostic;
-        let msg = "the `#[test]` attribute may only be used on a non-associated function";
-        let mut err = match item.kind {
-            // These were a warning before #92959 and need to continue being that to avoid breaking
-            // stable user code (#94508).
-            ast::ItemKind::MacCall(_) => diag.struct_span_warn(attr_sp, msg),
-            // `.forget_guarantee()` needed to get these two arms to match types. Because of how
-            // locally close the `.emit()` call is I'm comfortable with it, but if it can be
-            // reworked in the future to not need it, it'd be nice.
-            _ => diag.struct_span_err(attr_sp, msg).forget_guarantee(),
+        not_testable_error(cx, attr_sp, Some(&item));
+        return if is_stmt {
+            vec![Annotatable::Stmt(P(ast::Stmt {
+                id: ast::DUMMY_NODE_ID,
+                span: item.span,
+                kind: ast::StmtKind::Item(item),
+            }))]
+        } else {
+            vec![Annotatable::Item(item)]
         };
-        err.span_label(attr_sp, "the `#[test]` macro causes a function to be run on a test and has no effect on non-functions")
-            .span_label(item.span, format!("expected a non-associated function, found {} {}", item.kind.article(), item.kind.descr()))
-            .span_suggestion(attr_sp, "replace with conditional compilation to make the item only exist when tests are being run", "#[cfg(test)]", Applicability::MaybeIncorrect)
-            .emit();
-
-        return vec![Annotatable::Item(item)];
     };
 
     // has_*_signature will report any errors in the type so compilation
@@ -231,25 +240,29 @@ pub fn expand_test_or_bench(
         &item.ident,
     ));
 
-    let mut test_const = cx.item(
-        sp,
-        Ident::new(item.ident.name, sp),
-        thin_vec![
-            // #[cfg(test)]
-            cx.attr_nested_word(sym::cfg, sym::test, attr_sp),
-            // #[rustc_test_marker = "test_case_sort_key"]
-            cx.attr_name_value_str(sym::rustc_test_marker, test_path_symbol, attr_sp),
-        ],
-        // const $ident: test::TestDescAndFn =
-        ast::ItemKind::Const(
-            ast::Defaultness::Final,
-            cx.ty(sp, ast::TyKind::Path(None, test_path("TestDescAndFn"))),
-            // test::TestDescAndFn {
-            Some(
-                cx.expr_struct(
-                    sp,
-                    test_path("TestDescAndFn"),
-                    thin_vec![
+    let location_info = get_location_info(cx, &item);
+
+    let mut test_const =
+        cx.item(
+            sp,
+            Ident::new(item.ident.name, sp),
+            thin_vec![
+                // #[cfg(test)]
+                cx.attr_nested_word(sym::cfg, sym::test, attr_sp),
+                // #[rustc_test_marker = "test_case_sort_key"]
+                cx.attr_name_value_str(sym::rustc_test_marker, test_path_symbol, attr_sp),
+            ],
+            // const $ident: test::TestDescAndFn =
+            ast::ItemKind::Const(
+                ast::ConstItem {
+                    defaultness: ast::Defaultness::Final,
+                    ty: cx.ty(sp, ast::TyKind::Path(None, test_path("TestDescAndFn"))),
+                    // test::TestDescAndFn {
+                    expr: Some(
+                        cx.expr_struct(
+                            sp,
+                            test_path("TestDescAndFn"),
+                            thin_vec![
                         // desc: test::TestDesc {
                         field(
                             "desc",
@@ -267,19 +280,26 @@ pub fn expand_test_or_bench(
                                         ),
                                     ),
                                     // ignore: true | false
-                                    field(
-                                        "ignore",
-                                        cx.expr_bool(sp, should_ignore(&cx.sess, &item)),
-                                    ),
+                                    field("ignore", cx.expr_bool(sp, should_ignore(&item)),),
                                     // ignore_message: Some("...") | None
                                     field(
                                         "ignore_message",
-                                        if let Some(msg) = should_ignore_message(cx, &item) {
+                                        if let Some(msg) = should_ignore_message(&item) {
                                             cx.expr_some(sp, cx.expr_str(sp, msg))
                                         } else {
                                             cx.expr_none(sp)
                                         },
                                     ),
+                                    // source_file: <relative_path_of_source_file>
+                                    field("source_file", cx.expr_str(sp, location_info.0)),
+                                    // start_line: start line of the test fn identifier.
+                                    field("start_line", cx.expr_usize(sp, location_info.1)),
+                                    // start_col: start column of the test fn identifier.
+                                    field("start_col", cx.expr_usize(sp, location_info.2)),
+                                    // end_line: end line of the test fn identifier.
+                                    field("end_line", cx.expr_usize(sp, location_info.3)),
+                                    // end_col: end column of the test fn identifier.
+                                    field("end_col", cx.expr_usize(sp, location_info.4)),
                                     // compile_fail: true | false
                                     field("compile_fail", cx.expr_bool(sp, false)),
                                     // no_run: true | false
@@ -329,10 +349,12 @@ pub fn expand_test_or_bench(
                         // testfn: test::StaticTestFn(...) | test::StaticBenchFn(...)
                         field("testfn", test_fn), // }
                     ],
-                ), // }
+                        ), // }
+                    ),
+                }
+                .into(),
             ),
-        ),
-    );
+        );
     test_const = test_const.map(|mut tc| {
         tc.vis.kind = ast::VisibilityKind::Public;
         tc
@@ -364,6 +386,49 @@ pub fn expand_test_or_bench(
     }
 }
 
+fn not_testable_error(cx: &ExtCtxt<'_>, attr_sp: Span, item: Option<&ast::Item>) {
+    let diag = &cx.sess.parse_sess.span_diagnostic;
+    let msg = "the `#[test]` attribute may only be used on a non-associated function";
+    let mut err = match item.map(|i| &i.kind) {
+        // These were a warning before #92959 and need to continue being that to avoid breaking
+        // stable user code (#94508).
+        Some(ast::ItemKind::MacCall(_)) => diag.struct_span_warn(attr_sp, msg),
+        // `.forget_guarantee()` needed to get these two arms to match types. Because of how
+        // locally close the `.emit()` call is I'm comfortable with it, but if it can be
+        // reworked in the future to not need it, it'd be nice.
+        _ => diag.struct_span_err(attr_sp, msg).forget_guarantee(),
+    };
+    if let Some(item) = item {
+        err.span_label(
+            item.span,
+            format!(
+                "expected a non-associated function, found {} {}",
+                item.kind.article(),
+                item.kind.descr()
+            ),
+        );
+    }
+    err.span_label(attr_sp, "the `#[test]` macro causes a function to be run as a test and has no effect on non-functions")
+        .span_suggestion(attr_sp,
+            "replace with conditional compilation to make the item only exist when tests are being run",
+            "#[cfg(test)]",
+            Applicability::MaybeIncorrect)
+        .emit();
+}
+
+fn get_location_info(cx: &ExtCtxt<'_>, item: &ast::Item) -> (Symbol, usize, usize, usize, usize) {
+    let span = item.ident.span;
+    let (source_file, lo_line, lo_col, hi_line, hi_col) =
+        cx.sess.source_map().span_to_location_info(span);
+
+    let file_name = match source_file {
+        Some(sf) => sf.name.display(FileNameDisplayPreference::Remapped).to_string(),
+        None => "no-location".to_string(),
+    };
+
+    (Symbol::intern(&file_name), lo_line, lo_col, hi_line, hi_col)
+}
+
 fn item_path(mod_path: &[Ident], item_ident: &Ident) -> String {
     mod_path
         .iter()
@@ -378,12 +443,12 @@ enum ShouldPanic {
     Yes(Option<Symbol>),
 }
 
-fn should_ignore(sess: &Session, i: &ast::Item) -> bool {
-    sess.contains_name(&i.attrs, sym::ignore)
+fn should_ignore(i: &ast::Item) -> bool {
+    attr::contains_name(&i.attrs, sym::ignore)
 }
 
-fn should_ignore_message(cx: &ExtCtxt<'_>, i: &ast::Item) -> Option<Symbol> {
-    match cx.sess.find_by_name(&i.attrs, sym::ignore) {
+fn should_ignore_message(i: &ast::Item) -> Option<Symbol> {
+    match attr::find_by_name(&i.attrs, sym::ignore) {
         Some(attr) => {
             match attr.meta_item_list() {
                 // Handle #[ignore(bar = "foo")]
@@ -397,7 +462,7 @@ fn should_ignore_message(cx: &ExtCtxt<'_>, i: &ast::Item) -> Option<Symbol> {
 }
 
 fn should_panic(cx: &ExtCtxt<'_>, i: &ast::Item) -> ShouldPanic {
-    match cx.sess.find_by_name(&i.attrs, sym::should_panic) {
+    match attr::find_by_name(&i.attrs, sym::should_panic) {
         Some(attr) => {
             let sd = &cx.sess.parse_sess.span_diagnostic;
 
@@ -463,7 +528,7 @@ fn test_type(cx: &ExtCtxt<'_>) -> TestType {
 }
 
 fn has_test_signature(cx: &ExtCtxt<'_>, i: &ast::Item) -> bool {
-    let has_should_panic_attr = cx.sess.contains_name(&i.attrs, sym::should_panic);
+    let has_should_panic_attr = attr::contains_name(&i.attrs, sym::should_panic);
     let sd = &cx.sess.parse_sess.span_diagnostic;
     match &i.kind {
         ast::ItemKind::Fn(box ast::Fn { sig, generics, .. }) => {
diff --git a/compiler/rustc_builtin_macros/src/test_harness.rs b/compiler/rustc_builtin_macros/src/test_harness.rs
index d8e3db9e8..80f497333 100644
--- a/compiler/rustc_builtin_macros/src/test_harness.rs
+++ b/compiler/rustc_builtin_macros/src/test_harness.rs
@@ -37,7 +37,7 @@ struct TestCtxt<'a> {
 
 /// Traverse the crate, collecting all the test functions, eliding any
 /// existing main functions, and synthesizing a main test harness
-pub fn inject(sess: &Session, resolver: &mut dyn ResolverExpand, krate: &mut ast::Crate) {
+pub fn inject(krate: &mut ast::Crate, sess: &Session, resolver: &mut dyn ResolverExpand) {
     let span_diagnostic = sess.diagnostic();
     let panic_strategy = sess.panic_strategy();
     let platform_panic_strategy = sess.target.panic_strategy;
@@ -47,13 +47,13 @@ pub fn inject(sess: &Session, resolver: &mut dyn ResolverExpand, krate: &mut ast
     // unconditional, so that the attribute is still marked as used in
     // non-test builds.
     let reexport_test_harness_main =
-        sess.first_attr_value_str_by_name(&krate.attrs, sym::reexport_test_harness_main);
+        attr::first_attr_value_str_by_name(&krate.attrs, sym::reexport_test_harness_main);
 
     // Do this here so that the test_runner crate attribute gets marked as used
     // even in non-test builds
-    let test_runner = get_test_runner(sess, span_diagnostic, &krate);
+    let test_runner = get_test_runner(span_diagnostic, &krate);
 
-    if sess.opts.test {
+    if sess.is_test_crate() {
         let panic_strategy = match (panic_strategy, sess.opts.unstable_opts.panic_abort_tests) {
             (PanicStrategy::Abort, true) => PanicStrategy::Abort,
             (PanicStrategy::Abort, false) => {
@@ -123,7 +123,7 @@ impl<'a> MutVisitor for TestHarnessGenerator<'a> {
 
     fn flat_map_item(&mut self, i: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
         let mut item = i.into_inner();
-        if let Some(name) = get_test_name(&self.cx.ext_cx.sess, &item) {
+        if let Some(name) = get_test_name(&item) {
             debug!("this is a test item");
 
             let test = Test { span: item.span, ident: item.ident, name };
@@ -145,12 +145,12 @@ impl<'a> MutVisitor for TestHarnessGenerator<'a> {
 
 // Beware, this is duplicated in librustc_passes/entry.rs (with
 // `rustc_hir::Item`), so make sure to keep them in sync.
-fn entry_point_type(sess: &Session, item: &ast::Item, depth: usize) -> EntryPointType {
+fn entry_point_type(item: &ast::Item, depth: usize) -> EntryPointType {
     match item.kind {
         ast::ItemKind::Fn(..) => {
-            if sess.contains_name(&item.attrs, sym::start) {
+            if attr::contains_name(&item.attrs, sym::start) {
                 EntryPointType::Start
-            } else if sess.contains_name(&item.attrs, sym::rustc_main) {
+            } else if attr::contains_name(&item.attrs, sym::rustc_main) {
                 EntryPointType::RustcMainAttr
             } else if item.ident.name == sym::main {
                 if depth == 0 {
@@ -184,7 +184,7 @@ impl<'a> MutVisitor for EntryPointCleaner<'a> {
         // Remove any #[rustc_main] or #[start] from the AST so it doesn't
         // clash with the one we're going to add, but mark it as
         // #[allow(dead_code)] to avoid printing warnings.
-        let item = match entry_point_type(self.sess, &item, self.depth) {
+        let item = match entry_point_type(&item, self.depth) {
             EntryPointType::MainNamed | EntryPointType::RustcMainAttr | EntryPointType::Start => {
                 item.map(|ast::Item { id, ident, attrs, kind, vis, span, tokens }| {
                     let allow_dead_code = attr::mk_attr_nested_word(
@@ -373,16 +373,12 @@ fn mk_tests_slice(cx: &TestCtxt<'_>, sp: Span) -> P<ast::Expr> {
     )
 }
 
-fn get_test_name(sess: &Session, i: &ast::Item) -> Option<Symbol> {
-    sess.first_attr_value_str_by_name(&i.attrs, sym::rustc_test_marker)
+fn get_test_name(i: &ast::Item) -> Option<Symbol> {
+    attr::first_attr_value_str_by_name(&i.attrs, sym::rustc_test_marker)
 }
 
-fn get_test_runner(
-    sess: &Session,
-    sd: &rustc_errors::Handler,
-    krate: &ast::Crate,
-) -> Option<ast::Path> {
-    let test_attr = sess.find_by_name(&krate.attrs, sym::test_runner)?;
+fn get_test_runner(sd: &rustc_errors::Handler, krate: &ast::Crate) -> Option<ast::Path> {
+    let test_attr = attr::find_by_name(&krate.attrs, sym::test_runner)?;
     let meta_list = test_attr.meta_item_list()?;
     let span = test_attr.span;
     match &*meta_list {
diff --git a/compiler/rustc_builtin_macros/src/util.rs b/compiler/rustc_builtin_macros/src/util.rs
index 83812631c..9463a1418 100644
--- a/compiler/rustc_builtin_macros/src/util.rs
+++ b/compiler/rustc_builtin_macros/src/util.rs
@@ -1,4 +1,4 @@
-use rustc_ast::{AttrStyle, Attribute, MetaItem};
+use rustc_ast::{attr, AttrStyle, Attribute, MetaItem};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_feature::AttributeTemplate;
 use rustc_lint_defs::builtin::DUPLICATE_MACRO_ATTRIBUTES;
@@ -36,7 +36,7 @@ pub fn warn_on_duplicate_attribute(ecx: &ExtCtxt<'_>, item: &Annotatable, name:
         _ => None,
     };
     if let Some(attrs) = attrs {
-        if let Some(attr) = ecx.sess.find_by_name(attrs, name) {
+        if let Some(attr) = attr::find_by_name(attrs, name) {
             ecx.parse_sess().buffer_lint(
                 DUPLICATE_MACRO_ATTRIBUTES,
                 attr.span,
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml b/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml
new file mode 100644
index 000000000..5f5510a57
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml
@@ -0,0 +1,64 @@
+name: Abi-cafe
+
+on:
+  - push
+
+jobs:
+  abi_cafe:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}-${{ matrix.env.TARGET_TRIPLE }}
+      cancel-in-progress: true
+
+    defaults:
+      run:
+        shell: bash
+
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - os: ubuntu-latest
+            env:
+              TARGET_TRIPLE: x86_64-unknown-linux-gnu
+          - os: macos-latest
+            env:
+              TARGET_TRIPLE: x86_64-apple-darwin
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-msvc
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: build/cg_clif
+        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Set MinGW as the default toolchain
+      if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: rustup set default-host x86_64-pc-windows-gnu
+
+    - name: Use sparse cargo registry
+      run: |
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
+
+    - name: Prepare dependencies
+      run: ./y.rs prepare
+
+    - name: Build
+      run: ./y.rs build --sysroot none
+
+    - name: Test abi-cafe
+      env:
+        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
+      run: ./y.rs abi-cafe
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
index 9d3ed3ac5..98b34c65d 100644
--- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
@@ -22,7 +22,7 @@ jobs:
         rustfmt --check build_system/mod.rs
 
 
-  build:
+  test:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
 
@@ -114,63 +114,6 @@ jobs:
       run: ./y.rs test
 
 
-  abi_cafe:
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
-
-    defaults:
-      run:
-        shell: bash
-
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - os: ubuntu-latest
-            env:
-              TARGET_TRIPLE: x86_64-unknown-linux-gnu
-          - os: macos-latest
-            env:
-              TARGET_TRIPLE: x86_64-apple-darwin
-          - os: windows-latest
-            env:
-              TARGET_TRIPLE: x86_64-pc-windows-msvc
-          - os: windows-latest
-            env:
-              TARGET_TRIPLE: x86_64-pc-windows-gnu
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Cache cargo target dir
-      uses: actions/cache@v3
-      with:
-        path: build/cg_clif
-        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
-
-    - name: Set MinGW as the default toolchain
-      if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
-      run: rustup set default-host x86_64-pc-windows-gnu
-
-    - name: Use sparse cargo registry
-      run: |
-        cat >> ~/.cargo/config.toml <<EOF
-        [unstable]
-        sparse-registry = true
-        EOF
-
-    - name: Prepare dependencies
-      run: ./y.rs prepare
-
-    - name: Build
-      run: ./y.rs build --sysroot none
-
-    - name: Test abi-cafe
-      env:
-        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
-      run: ./y.rs abi-cafe
-
-
   bench:
     runs-on: ubuntu-latest
     timeout-minutes: 60
diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index 50249ea1b..87e4ac266 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -57,18 +57,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f3d54eab028f5805ae3b26fd60eca3f3a9cfb76b989d9bab173be3f61356cc3"
+checksum = "a7379abaacee0f14abf3204a7606118f0465785252169d186337bcb75030815a"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2be1d5f2c3cca1efb691844bc1988b89c77291f13f778499a3f3c0cf49c0ed61"
+checksum = "9489fa336927df749631f1008007ced2871068544f40a202ce6d93fbf2366a7b"
 dependencies = [
  "arrayvec",
  "bumpalo",
@@ -87,30 +87,30 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9b1b1089750ce4005893af7ee00bb08a2cf1c9779999c0f7164cbc8ad2e0d2"
+checksum = "05bbb67da91ec721ed57cef2f7c5ef7728e1cd9bde9ffd3ef8601022e73e3239"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc5fbaec51de47297fd7304986fd53c8c0030abbe69728a60d72e1c63559318d"
+checksum = "418ecb2f36032f6665dc1a5e2060a143dbab41d83b784882e97710e890a7a16d"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dab984c94593f876090fae92e984bdcc74d9b1acf740ab5f79036001c65cba13"
+checksum = "7cf583f7b093f291005f9fb1323e2c37f6ee4c7909e39ce016b2e8360d461705"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e0cb3102d21a2fe5f3210af608748ddd0cd09825ac12d42dc56ed5ed8725fe0"
+checksum = "7d361ed0373cf5f086b49c499aa72227b646a64f899f32e34312f97c0fadff75"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -120,15 +120,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.92.0"
+version = "0.93.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72101dd1f441d629735143c41e00b3428f9267738176983ef588ff43382af0a0"
+checksum = "649782a39ce99798dd6b4029e2bb318a2fbeaade1b4fa25330763c10c65bc358"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6557f8ce44d498777f2495aa58d9692a4a37d6f84aa445750d666cef770b6a5c"
+checksum = "9c9909222db472fcc98d9e4e7192fa9d064dac63a3fa657df8c6daae86fb2604"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -145,9 +145,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88807e1c0c47ec02fe433333ccbe56b480425418b1470e333205e11650697d72"
+checksum = "68689b83e52e605ba48652882d3fccc2e2e136abf139eb64ae667888ba0d52f8"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -155,9 +155,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c22b0d9fcbe3fc5a1af9e7021b44ce42b930bcefac446ce22e02e8f9a0d67120"
+checksum = "f98e4e99a353703475d5acb402b9c13482d41d8a4008b352559bd560afb90363"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -166,9 +166,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.92.0"
+version = "0.93.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "341375758d7c3fedc0b5315f552e6f0feac46baf87c450a15e9455ef47c2b261"
+checksum = "b7a006ce1d8dd11df67567d8673e5920f3a56441812aed52a007ffce8f1b20e9"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -235,9 +235,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "1.9.2"
+version = "1.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
 dependencies = [
  "autocfg",
  "hashbrown",
@@ -333,6 +333,7 @@ dependencies = [
  "cranelift-frontend",
  "cranelift-jit",
  "cranelift-module",
+ "cranelift-native",
  "cranelift-object",
  "gimli",
  "indexmap",
@@ -381,9 +382,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "5.0.0"
+version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08fcba5ebd96da2a9f0747ab6337fe9788adfb3f63fa2c180520d665562d257e"
+checksum = "ec1fd0f0dd79e7cc0f55b102e320d7c77ab76cd272008a8fd98e25b5777e2636"
 dependencies = [
  "cfg-if",
  "libc",
diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml
index 34117c288..5dadcaaec 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/Cargo.toml
@@ -15,19 +15,17 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.92", features = ["unwind", "all-arch"] }
-cranelift-frontend = { version = "0.92" }
-cranelift-module = { version = "0.92" }
-# NOTE vendored as src/cranelift_native.rs
-# FIXME revert back to the external crate with Cranelift 0.93
-#cranelift-native = { version = "0.92" }
-cranelift-jit = { version = "0.92", optional = true }
-cranelift-object = { version = "0.92" }
+cranelift-codegen = { version = "0.93", features = ["unwind", "all-arch"] }
+cranelift-frontend = { version = "0.93" }
+cranelift-module = { version = "0.93" }
+cranelift-native = { version = "0.93" }
+cranelift-jit = { version = "0.93", optional = true }
+cranelift-object = { version = "0.93" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.26.0", default-features = false, features = ["write"]}
 object = { version = "0.29.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
-indexmap = "1.9.1"
+indexmap = "1.9.3"
 libloading = { version = "0.7.3", optional = true }
 once_cell = "1.10.0"
 smallvec = "1.8.1"
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
index b7e0b68a2..f21507629 100644
--- a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
@@ -50,9 +50,9 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.86"
+version = "0.1.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dae98c88e576098d7ab13ebcb40cc43e5114b2beafe61a87cda9200649ff205"
+checksum = "9fc9c2080d347a2c316518840ac9194644a9993dfa1e9778ef38979a339f5d8b"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -117,21 +117,20 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.2.6"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
 dependencies = [
  "compiler_builtins",
- "libc",
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
 ]
 
 [[package]]
 name = "libc"
-version = "0.2.139"
+version = "0.2.140"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -282,10 +281,8 @@ dependencies = [
 name = "test"
 version = "0.0.0"
 dependencies = [
- "cfg-if",
  "core",
  "getopts",
- "libc",
  "panic_abort",
  "panic_unwind",
  "proc_macro",
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml
index d0e5fc4a3..8219e6b6c 100644
--- a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml
@@ -8,7 +8,7 @@ alloc = { path = "./sysroot_src/library/alloc" }
 std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] }
 test = { path = "./sysroot_src/library/test" }
 
-compiler_builtins = { version = "0.1.39", default-features = false, features = ["no-asm"] }
+compiler_builtins = { version = "0.1.87", default-features = false, features = ["no-asm"] }
 
 [patch.crates-io]
 rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" }
diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
index 2e04f2c68..76b602fe7 100644
--- a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
@@ -46,13 +46,15 @@ pub(crate) fn build_sysroot(
         let wrapper_name = wrapper_base_name.replace("____", wrapper);
 
         let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
+        let wrapper_path = DIST_DIR.to_path(dirs).join(&wrapper_name);
         build_cargo_wrapper_cmd
             .env("TOOLCHAIN_NAME", toolchain_name.clone())
             .arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
             .arg("-o")
-            .arg(DIST_DIR.to_path(dirs).join(wrapper_name))
+            .arg(&wrapper_path)
             .arg("-Cstrip=debuginfo");
         spawn_and_wait(build_cargo_wrapper_cmd);
+        try_hard_link(wrapper_path, BIN_DIR.to_path(dirs).join(wrapper_name));
     }
 
     let host = build_sysroot_for_triple(
@@ -247,6 +249,7 @@ fn build_clif_sysroot_for_triple(
     if channel == "release" {
         build_cmd.arg("--release");
     }
+    build_cmd.arg("--locked");
     build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
     if compiler.triple.contains("apple") {
         build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed");
diff --git a/compiler/rustc_codegen_cranelift/build_system/prepare.rs b/compiler/rustc_codegen_cranelift/build_system/prepare.rs
index 50b1b7836..6769e42d4 100644
--- a/compiler/rustc_codegen_cranelift/build_system/prepare.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/prepare.rs
@@ -6,6 +6,7 @@ use std::process::Command;
 use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
 use super::path::{Dirs, RelPath};
 use super::rustc_info::{get_default_sysroot, get_rustc_version};
+use super::tests::LIBCORE_TESTS_SRC;
 use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait};
 
 pub(crate) fn prepare(dirs: &Dirs) {
@@ -13,8 +14,10 @@ pub(crate) fn prepare(dirs: &Dirs) {
 
     spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", "rustc", dirs));
 
-    prepare_sysroot(dirs);
+    prepare_stdlib(dirs);
     spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", "rustc", dirs));
+
+    prepare_coretests(dirs);
     spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", "rustc", dirs));
 
     super::tests::RAND_REPO.fetch(dirs);
@@ -25,11 +28,11 @@ pub(crate) fn prepare(dirs: &Dirs) {
     spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", "rustc", dirs));
 }
 
-fn prepare_sysroot(dirs: &Dirs) {
+fn prepare_stdlib(dirs: &Dirs) {
     let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
     assert!(sysroot_src_orig.exists());
 
-    eprintln!("[COPY] sysroot src");
+    eprintln!("[COPY] stdlib src");
 
     // FIXME ensure builds error out or update the copy if any of the files copied here change
     BUILD_SYSROOT.ensure_fresh(dirs);
@@ -47,7 +50,25 @@ fn prepare_sysroot(dirs: &Dirs) {
     eprintln!("[GIT] init");
     init_git_repo(&SYSROOT_SRC.to_path(dirs));
 
-    apply_patches(dirs, "sysroot", &SYSROOT_SRC.to_path(dirs));
+    apply_patches(dirs, "stdlib", &SYSROOT_SRC.to_path(dirs));
+}
+
+fn prepare_coretests(dirs: &Dirs) {
+    let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
+    assert!(sysroot_src_orig.exists());
+
+    eprintln!("[COPY] coretests src");
+
+    fs::create_dir_all(LIBCORE_TESTS_SRC.to_path(dirs)).unwrap();
+    copy_dir_recursively(
+        &sysroot_src_orig.join("library/core/tests"),
+        &LIBCORE_TESTS_SRC.to_path(dirs),
+    );
+
+    eprintln!("[GIT] init");
+    init_git_repo(&LIBCORE_TESTS_SRC.to_path(dirs));
+
+    apply_patches(dirs, "coretests", &LIBCORE_TESTS_SRC.to_path(dirs));
 }
 
 pub(crate) struct GitRepo {
diff --git a/compiler/rustc_codegen_cranelift/build_system/tests.rs b/compiler/rustc_codegen_cranelift/build_system/tests.rs
index e9486888f..261948a69 100644
--- a/compiler/rustc_codegen_cranelift/build_system/tests.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/tests.rs
@@ -1,4 +1,4 @@
-use super::build_sysroot::{self, SYSROOT_SRC};
+use super::build_sysroot;
 use super::config;
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
@@ -94,40 +94,42 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]),
 ];
 
+// FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is
+// fixed.
 pub(crate) static RAND_REPO: GitRepo =
-    GitRepo::github("rust-random", "rand", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", "rand");
+    GitRepo::github("rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", "rand");
 
 pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand");
 
 pub(crate) static REGEX_REPO: GitRepo =
-    GitRepo::github("rust-lang", "regex", "341f207c1071f7290e3f228c710817c280c8dca1", "regex");
+    GitRepo::github("rust-lang", "regex", "a9b2e02352db92ce1f6e5b7ecd41b8bbffbe161a", "regex");
 
 pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir(), "regex");
 
 pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
     "rust-lang",
     "portable-simd",
-    "582239ac3b32007613df04d7ffa78dc30f4c5645",
+    "9bd30e77b3a3c699af102ebb3df0f6110f8aa02e",
     "portable-simd",
 );
 
 pub(crate) static PORTABLE_SIMD: CargoProject =
     CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd");
 
-pub(crate) static LIBCORE_TESTS: CargoProject =
-    CargoProject::new(&SYSROOT_SRC.join("library/core/tests"), "core_tests");
+pub(crate) static LIBCORE_TESTS_SRC: RelPath = RelPath::DOWNLOAD.join("coretests_src");
+
+pub(crate) static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests");
 
 const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::custom("test.rust-random/rand", &|runner| {
         RAND.clean(&runner.dirs);
 
         if runner.is_native {
-            eprintln!("[TEST] rust-random/rand");
             let mut test_cmd = RAND.test(&runner.target_compiler, &runner.dirs);
-            test_cmd.arg("--workspace");
+            test_cmd.arg("--workspace").arg("--").arg("-q");
             spawn_and_wait(test_cmd);
         } else {
-            eprintln!("[AOT] rust-random/rand");
+            eprintln!("Cross-Compiling: Not running tests");
             let mut build_cmd = RAND.build(&runner.target_compiler, &runner.dirs);
             build_cmd.arg("--workspace").arg("--tests");
             spawn_and_wait(build_cmd);
@@ -137,7 +139,9 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
         LIBCORE_TESTS.clean(&runner.dirs);
 
         if runner.is_native {
-            spawn_and_wait(LIBCORE_TESTS.test(&runner.target_compiler, &runner.dirs));
+            let mut test_cmd = LIBCORE_TESTS.test(&runner.target_compiler, &runner.dirs);
+            test_cmd.arg("--").arg("-q");
+            spawn_and_wait(test_cmd);
         } else {
             eprintln!("Cross-Compiling: Not running tests");
             let mut build_cmd = LIBCORE_TESTS.build(&runner.target_compiler, &runner.dirs);
@@ -148,18 +152,13 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::custom("test.regex-shootout-regex-dna", &|runner| {
         REGEX.clean(&runner.dirs);
 
-        // newer aho_corasick versions throw a deprecation warning
-        let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
-
         let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs);
         build_cmd.arg("--example").arg("shootout-regex-dna");
-        build_cmd.env("RUSTFLAGS", lint_rust_flags.clone());
         spawn_and_wait(build_cmd);
 
         if runner.is_native {
             let mut run_cmd = REGEX.run(&runner.target_compiler, &runner.dirs);
             run_cmd.arg("--example").arg("shootout-regex-dna");
-            run_cmd.env("RUSTFLAGS", lint_rust_flags);
 
             let input = fs::read_to_string(
                 REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-input.txt"),
@@ -171,13 +170,6 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
             .unwrap();
 
             let output = spawn_and_wait_with_input(run_cmd, input);
-            // Make sure `[codegen mono items] start` doesn't poison the diff
-            let output = output
-                .lines()
-                .filter(|line| !line.contains("codegen mono items"))
-                .chain(Some("")) // This just adds the trailing newline
-                .collect::<Vec<&str>>()
-                .join("\r\n");
 
             let output_matches = expected.lines().eq(output.lines());
             if !output_matches {
@@ -192,27 +184,14 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::custom("test.regex", &|runner| {
         REGEX.clean(&runner.dirs);
 
-        // newer aho_corasick versions throw a deprecation warning
-        let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
-
         if runner.is_native {
             let mut run_cmd = REGEX.test(&runner.target_compiler, &runner.dirs);
-            run_cmd.args([
-                "--tests",
-                "--",
-                "--exclude-should-panic",
-                "--test-threads",
-                "1",
-                "-Zunstable-options",
-                "-q",
-            ]);
-            run_cmd.env("RUSTFLAGS", lint_rust_flags);
+            run_cmd.args(["--workspace", "--", "-q"]);
             spawn_and_wait(run_cmd);
         } else {
             eprintln!("Cross-Compiling: Not running tests");
             let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs);
             build_cmd.arg("--tests");
-            build_cmd.env("RUSTFLAGS", lint_rust_flags.clone());
             spawn_and_wait(build_cmd);
         }
     }),
diff --git a/compiler/rustc_codegen_cranelift/example/alloc_example.rs b/compiler/rustc_codegen_cranelift/example/alloc_example.rs
index bc1594d82..4ede2fe4e 100644
--- a/compiler/rustc_codegen_cranelift/example/alloc_example.rs
+++ b/compiler/rustc_codegen_cranelift/example/alloc_example.rs
@@ -1,4 +1,4 @@
-#![feature(start, core_intrinsics, alloc_error_handler, box_syntax)]
+#![feature(start, core_intrinsics, alloc_error_handler)]
 #![no_std]
 
 extern crate alloc;
@@ -29,7 +29,7 @@ fn alloc_error_handler(_: alloc::alloc::Layout) -> ! {
 
 #[start]
 fn main(_argc: isize, _argv: *const *const u8) -> isize {
-    let world: Box<&str> = box "Hello World!\0";
+    let world: Box<&str> = Box::new("Hello World!\0");
     unsafe {
         puts(*world as *const str as *const u8);
     }
diff --git a/compiler/rustc_codegen_cranelift/example/alloc_system.rs b/compiler/rustc_codegen_cranelift/example/alloc_system.rs
index 50261c193..e64daf96b 100644
--- a/compiler/rustc_codegen_cranelift/example/alloc_system.rs
+++ b/compiler/rustc_codegen_cranelift/example/alloc_system.rs
@@ -1,12 +1,6 @@
-// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
+// SPDX-License-Identifier: MIT OR Apache-2.0
+// SPDX-FileCopyrightText: The Rust Project Developers (see https://thanks.rust-lang.org)
+
 #![no_std]
 
 pub struct System;
diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs
index 1f9db1eb2..73b83b89f 100644
--- a/compiler/rustc_codegen_cranelift/example/mini_core.rs
+++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs
@@ -518,6 +518,17 @@ pub struct Box<T: ?Sized>(Unique<T>, ());
 
 impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<Box<U>> for Box<T> {}
 
+impl<T> Box<T> {
+    pub fn new(val: T) -> Box<T> {
+        unsafe {
+            let size = intrinsics::size_of::<T>();
+            let ptr = libc::malloc(size);
+            intrinsics::copy(&val as *const T as *const u8, ptr, size);
+            Box(Unique { pointer: NonNull(ptr as *const T), _marker: PhantomData }, ())
+        }
+    }
+}
+
 impl<T: ?Sized> Drop for Box<T> {
     fn drop(&mut self) {
         // drop is currently performed by compiler.
diff --git a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
index c00f8a2e0..6ad3268e7 100644
--- a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
+++ b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
@@ -1,16 +1,16 @@
-#![feature(no_core, lang_items, never_type, linkage, extern_types, thread_local, box_syntax)]
+#![feature(no_core, lang_items, never_type, linkage, extern_types, thread_local, repr_simd)]
 #![no_core]
 #![allow(dead_code, non_camel_case_types)]
 
 extern crate mini_core;
 
-use mini_core::*;
 use mini_core::libc::*;
+use mini_core::*;
 
 macro_rules! assert {
     ($e:expr) => {
         if !$e {
-            panic(stringify!(! $e));
+            panic(stringify!(!$e));
         }
     };
 }
@@ -20,7 +20,7 @@ macro_rules! assert_eq {
         if $l != $r {
             panic(stringify!($l != $r));
         }
-    }
+    };
 }
 
 #[lang = "termination"]
@@ -96,9 +96,15 @@ fn start<T: Termination + 'static>(
     _sigpipe: u8,
 ) -> isize {
     if argc == 3 {
-        unsafe { puts(*argv as *const i8); }
-        unsafe { puts(*((argv as usize + intrinsics::size_of::<*const u8>()) as *const *const i8)); }
-        unsafe { puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8)); }
+        unsafe {
+            puts(*argv as *const i8);
+        }
+        unsafe {
+            puts(*((argv as usize + intrinsics::size_of::<*const u8>()) as *const *const i8));
+        }
+        unsafe {
+            puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8));
+        }
     }
 
     main().report() as isize
@@ -107,7 +113,6 @@ fn start<T: Termination + 'static>(
 static mut NUM: u8 = 6 * 7;
 static NUM_REF: &'static u8 = unsafe { &NUM };
 
-
 unsafe fn zeroed<T>() -> T {
     let mut uninit = MaybeUninit { uninit: () };
     intrinsics::write_bytes(&mut uninit.value.value as *mut T, 0, 1);
@@ -144,10 +149,7 @@ extern "C" fn bool_struct_in_11(_arg0: bool_11) {}
 
 #[allow(unreachable_code)] // FIXME false positive
 fn main() {
-    take_unique(Unique {
-        pointer: unsafe { NonNull(1 as *mut ()) },
-        _marker: PhantomData,
-    });
+    take_unique(Unique { pointer: unsafe { NonNull(1 as *mut ()) }, _marker: PhantomData });
     take_f32(0.1);
 
     call_return_u128_pair();
@@ -178,7 +180,7 @@ fn main() {
         let ptr: *const i8 = hello as *const [u8] as *const i8;
         puts(ptr);
 
-        let world: Box<&str> = box "World!\0";
+        let world: Box<&str> = Box::new("World!\0");
         puts(*world as *const str as *const i8);
         world as Box<dyn SomeTrait>;
 
@@ -202,17 +204,17 @@ fn main() {
         assert_eq!(intrinsics::size_of_val(&0u32) as u8, 4);
 
         assert_eq!(intrinsics::min_align_of::<u16>() as u8, 2);
-        assert_eq!(intrinsics::min_align_of_val(&a) as u8, intrinsics::min_align_of::<&str>() as u8);
+        assert_eq!(
+            intrinsics::min_align_of_val(&a) as u8,
+            intrinsics::min_align_of::<&str>() as u8
+        );
 
         assert!(!intrinsics::needs_drop::<u8>());
         assert!(!intrinsics::needs_drop::<[u8]>());
         assert!(intrinsics::needs_drop::<NoisyDrop>());
         assert!(intrinsics::needs_drop::<NoisyDropUnsized>());
 
-        Unique {
-            pointer: NonNull(1 as *mut &str),
-            _marker: PhantomData,
-        } as Unique<dyn SomeTrait>;
+        Unique { pointer: NonNull(1 as *mut &str), _marker: PhantomData } as Unique<dyn SomeTrait>;
 
         struct MyDst<T: ?Sized>(T);
 
@@ -238,19 +240,17 @@ fn main() {
         }
     }
 
-    let _ = box NoisyDrop {
-        text: "Boxed outer got dropped!\0",
-        inner: NoisyDropInner,
-    } as Box<dyn SomeTrait>;
+    let _ = Box::new(NoisyDrop { text: "Boxed outer got dropped!\0", inner: NoisyDropInner })
+        as Box<dyn SomeTrait>;
 
     const FUNC_REF: Option<fn()> = Some(main);
     match FUNC_REF {
-        Some(_) => {},
+        Some(_) => {}
         None => assert!(false),
     }
 
     match Ordering::Less {
-        Ordering::Less => {},
+        Ordering::Less => {}
         _ => assert!(false),
     }
 
@@ -266,19 +266,21 @@ fn main() {
 
     #[cfg(not(any(jit, windows)))]
     {
-        extern {
+        extern "C" {
             #[linkage = "extern_weak"]
             static ABC: *const u8;
         }
 
         {
-            extern {
+            extern "C" {
                 #[linkage = "extern_weak"]
                 static ABC: *const u8;
             }
         }
 
-        unsafe { assert_eq!(ABC as usize, 0); }
+        unsafe {
+            assert_eq!(ABC as usize, 0);
+        }
     }
 
     &mut (|| Some(0 as *const ())) as &mut dyn FnMut() -> Option<*const ()>;
@@ -339,7 +341,13 @@ fn main() {
 
         assert_eq!(unsafe { intrinsics::size_of_val(x) }, 0);
         assert_eq!(unsafe { intrinsics::min_align_of_val(x) }, 1);
-}
+    }
+
+    #[repr(simd)]
+    struct V([f64; 2]);
+
+    let f = V([0.0, 1.0]);
+    let _a = f.0[0];
 }
 
 #[cfg(all(not(jit), target_arch = "x86_64", any(target_os = "linux", target_os = "darwin")))]
@@ -392,13 +400,10 @@ extern "C" {
         native: *mut pthread_t,
         attr: *const pthread_attr_t,
         f: extern "C" fn(_: *mut c_void) -> *mut c_void,
-        value: *mut c_void
+        value: *mut c_void,
     ) -> c_int;
 
-    fn pthread_join(
-        native: pthread_t,
-        value: *mut *mut c_void
-    ) -> c_int;
+    fn pthread_join(native: pthread_t, value: *mut *mut c_void) -> c_int;
 }
 
 type DWORD = u32;
@@ -410,10 +415,7 @@ type HANDLE = *mut c_void;
 #[link(name = "msvcrt")]
 #[cfg(windows)]
 extern "C" {
-    fn WaitForSingleObject(
-        hHandle: LPVOID,
-        dwMilliseconds: DWORD
-    ) -> DWORD;
+    fn WaitForSingleObject(hHandle: LPVOID, dwMilliseconds: DWORD) -> DWORD;
 
     fn CreateThread(
         lpThreadAttributes: LPVOID, // Technically LPSECURITY_ATTRIBUTES, but we don't use it anyway
@@ -421,7 +423,7 @@ extern "C" {
         lpStartAddress: extern "C" fn(_: *mut c_void) -> *mut c_void,
         lpParameter: LPVOID,
         dwCreationFlags: DWORD,
-        lpThreadId: LPDWORD
+        lpThreadId: LPDWORD,
     ) -> HANDLE;
 }
 
@@ -447,9 +449,7 @@ impl Thread {
                 assert!(false);
             }
 
-            Thread {
-                handle: thread,
-            }
+            Thread { handle: thread }
         }
 
         #[cfg(windows)]
@@ -460,13 +460,10 @@ impl Thread {
                 assert!(false);
             }
 
-            Thread {
-                handle,
-            }
+            Thread { handle }
         }
     }
 
-
     unsafe fn join(self) {
         #[cfg(unix)]
         {
@@ -483,16 +480,15 @@ impl Thread {
     }
 }
 
-
-
-
 #[thread_local]
 #[cfg(not(jit))]
 static mut TLS: u8 = 42;
 
 #[cfg(not(jit))]
 extern "C" fn mutate_tls(_: *mut c_void) -> *mut c_void {
-    unsafe { TLS = 0; }
+    unsafe {
+        TLS = 0;
+    }
     0 as *mut c_void
 }
 
@@ -531,44 +527,267 @@ pub enum E1 {
 pub enum E2<X> {
     V1 { f: bool },
 
-    /*_00*/ _01(X), _02(X), _03(X), _04(X), _05(X), _06(X), _07(X),
-    _08(X), _09(X), _0A(X), _0B(X), _0C(X), _0D(X), _0E(X), _0F(X),
-    _10(X), _11(X), _12(X), _13(X), _14(X), _15(X), _16(X), _17(X),
-    _18(X), _19(X), _1A(X), _1B(X), _1C(X), _1D(X), _1E(X), _1F(X),
-    _20(X), _21(X), _22(X), _23(X), _24(X), _25(X), _26(X), _27(X),
-    _28(X), _29(X), _2A(X), _2B(X), _2C(X), _2D(X), _2E(X), _2F(X),
-    _30(X), _31(X), _32(X), _33(X), _34(X), _35(X), _36(X), _37(X),
-    _38(X), _39(X), _3A(X), _3B(X), _3C(X), _3D(X), _3E(X), _3F(X),
-    _40(X), _41(X), _42(X), _43(X), _44(X), _45(X), _46(X), _47(X),
-    _48(X), _49(X), _4A(X), _4B(X), _4C(X), _4D(X), _4E(X), _4F(X),
-    _50(X), _51(X), _52(X), _53(X), _54(X), _55(X), _56(X), _57(X),
-    _58(X), _59(X), _5A(X), _5B(X), _5C(X), _5D(X), _5E(X), _5F(X),
-    _60(X), _61(X), _62(X), _63(X), _64(X), _65(X), _66(X), _67(X),
-    _68(X), _69(X), _6A(X), _6B(X), _6C(X), _6D(X), _6E(X), _6F(X),
-    _70(X), _71(X), _72(X), _73(X), _74(X), _75(X), _76(X), _77(X),
-    _78(X), _79(X), _7A(X), _7B(X), _7C(X), _7D(X), _7E(X), _7F(X),
-    _80(X), _81(X), _82(X), _83(X), _84(X), _85(X), _86(X), _87(X),
-    _88(X), _89(X), _8A(X), _8B(X), _8C(X), _8D(X), _8E(X), _8F(X),
-    _90(X), _91(X), _92(X), _93(X), _94(X), _95(X), _96(X), _97(X),
-    _98(X), _99(X), _9A(X), _9B(X), _9C(X), _9D(X), _9E(X), _9F(X),
-    _A0(X), _A1(X), _A2(X), _A3(X), _A4(X), _A5(X), _A6(X), _A7(X),
-    _A8(X), _A9(X), _AA(X), _AB(X), _AC(X), _AD(X), _AE(X), _AF(X),
-    _B0(X), _B1(X), _B2(X), _B3(X), _B4(X), _B5(X), _B6(X), _B7(X),
-    _B8(X), _B9(X), _BA(X), _BB(X), _BC(X), _BD(X), _BE(X), _BF(X),
-    _C0(X), _C1(X), _C2(X), _C3(X), _C4(X), _C5(X), _C6(X), _C7(X),
-    _C8(X), _C9(X), _CA(X), _CB(X), _CC(X), _CD(X), _CE(X), _CF(X),
-    _D0(X), _D1(X), _D2(X), _D3(X), _D4(X), _D5(X), _D6(X), _D7(X),
-    _D8(X), _D9(X), _DA(X), _DB(X), _DC(X), _DD(X), _DE(X), _DF(X),
-    _E0(X), _E1(X), _E2(X), _E3(X), _E4(X), _E5(X), _E6(X), _E7(X),
-    _E8(X), _E9(X), _EA(X), _EB(X), _EC(X), _ED(X), _EE(X), _EF(X),
-    _F0(X), _F1(X), _F2(X), _F3(X), _F4(X), _F5(X), _F6(X), _F7(X),
-    _F8(X), _F9(X), _FA(X), _FB(X), _FC(X), _FD(X), _FE(X), _FF(X),
+    /*_00*/ _01(X),
+    _02(X),
+    _03(X),
+    _04(X),
+    _05(X),
+    _06(X),
+    _07(X),
+    _08(X),
+    _09(X),
+    _0A(X),
+    _0B(X),
+    _0C(X),
+    _0D(X),
+    _0E(X),
+    _0F(X),
+    _10(X),
+    _11(X),
+    _12(X),
+    _13(X),
+    _14(X),
+    _15(X),
+    _16(X),
+    _17(X),
+    _18(X),
+    _19(X),
+    _1A(X),
+    _1B(X),
+    _1C(X),
+    _1D(X),
+    _1E(X),
+    _1F(X),
+    _20(X),
+    _21(X),
+    _22(X),
+    _23(X),
+    _24(X),
+    _25(X),
+    _26(X),
+    _27(X),
+    _28(X),
+    _29(X),
+    _2A(X),
+    _2B(X),
+    _2C(X),
+    _2D(X),
+    _2E(X),
+    _2F(X),
+    _30(X),
+    _31(X),
+    _32(X),
+    _33(X),
+    _34(X),
+    _35(X),
+    _36(X),
+    _37(X),
+    _38(X),
+    _39(X),
+    _3A(X),
+    _3B(X),
+    _3C(X),
+    _3D(X),
+    _3E(X),
+    _3F(X),
+    _40(X),
+    _41(X),
+    _42(X),
+    _43(X),
+    _44(X),
+    _45(X),
+    _46(X),
+    _47(X),
+    _48(X),
+    _49(X),
+    _4A(X),
+    _4B(X),
+    _4C(X),
+    _4D(X),
+    _4E(X),
+    _4F(X),
+    _50(X),
+    _51(X),
+    _52(X),
+    _53(X),
+    _54(X),
+    _55(X),
+    _56(X),
+    _57(X),
+    _58(X),
+    _59(X),
+    _5A(X),
+    _5B(X),
+    _5C(X),
+    _5D(X),
+    _5E(X),
+    _5F(X),
+    _60(X),
+    _61(X),
+    _62(X),
+    _63(X),
+    _64(X),
+    _65(X),
+    _66(X),
+    _67(X),
+    _68(X),
+    _69(X),
+    _6A(X),
+    _6B(X),
+    _6C(X),
+    _6D(X),
+    _6E(X),
+    _6F(X),
+    _70(X),
+    _71(X),
+    _72(X),
+    _73(X),
+    _74(X),
+    _75(X),
+    _76(X),
+    _77(X),
+    _78(X),
+    _79(X),
+    _7A(X),
+    _7B(X),
+    _7C(X),
+    _7D(X),
+    _7E(X),
+    _7F(X),
+    _80(X),
+    _81(X),
+    _82(X),
+    _83(X),
+    _84(X),
+    _85(X),
+    _86(X),
+    _87(X),
+    _88(X),
+    _89(X),
+    _8A(X),
+    _8B(X),
+    _8C(X),
+    _8D(X),
+    _8E(X),
+    _8F(X),
+    _90(X),
+    _91(X),
+    _92(X),
+    _93(X),
+    _94(X),
+    _95(X),
+    _96(X),
+    _97(X),
+    _98(X),
+    _99(X),
+    _9A(X),
+    _9B(X),
+    _9C(X),
+    _9D(X),
+    _9E(X),
+    _9F(X),
+    _A0(X),
+    _A1(X),
+    _A2(X),
+    _A3(X),
+    _A4(X),
+    _A5(X),
+    _A6(X),
+    _A7(X),
+    _A8(X),
+    _A9(X),
+    _AA(X),
+    _AB(X),
+    _AC(X),
+    _AD(X),
+    _AE(X),
+    _AF(X),
+    _B0(X),
+    _B1(X),
+    _B2(X),
+    _B3(X),
+    _B4(X),
+    _B5(X),
+    _B6(X),
+    _B7(X),
+    _B8(X),
+    _B9(X),
+    _BA(X),
+    _BB(X),
+    _BC(X),
+    _BD(X),
+    _BE(X),
+    _BF(X),
+    _C0(X),
+    _C1(X),
+    _C2(X),
+    _C3(X),
+    _C4(X),
+    _C5(X),
+    _C6(X),
+    _C7(X),
+    _C8(X),
+    _C9(X),
+    _CA(X),
+    _CB(X),
+    _CC(X),
+    _CD(X),
+    _CE(X),
+    _CF(X),
+    _D0(X),
+    _D1(X),
+    _D2(X),
+    _D3(X),
+    _D4(X),
+    _D5(X),
+    _D6(X),
+    _D7(X),
+    _D8(X),
+    _D9(X),
+    _DA(X),
+    _DB(X),
+    _DC(X),
+    _DD(X),
+    _DE(X),
+    _DF(X),
+    _E0(X),
+    _E1(X),
+    _E2(X),
+    _E3(X),
+    _E4(X),
+    _E5(X),
+    _E6(X),
+    _E7(X),
+    _E8(X),
+    _E9(X),
+    _EA(X),
+    _EB(X),
+    _EC(X),
+    _ED(X),
+    _EE(X),
+    _EF(X),
+    _F0(X),
+    _F1(X),
+    _F2(X),
+    _F3(X),
+    _F4(X),
+    _F5(X),
+    _F6(X),
+    _F7(X),
+    _F8(X),
+    _F9(X),
+    _FA(X),
+    _FB(X),
+    _FC(X),
+    _FD(X),
+    _FE(X),
+    _FF(X),
 
     V3,
     V4,
 }
 
-fn check_niche_behavior () {
+fn check_niche_behavior() {
     if let E1::V2 { .. } = (E1::V1 { f: true }) {
         intrinsics::abort();
     }
diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs
index 8481d9c39..e34b35d5c 100644
--- a/compiler/rustc_codegen_cranelift/example/std_example.rs
+++ b/compiler/rustc_codegen_cranelift/example/std_example.rs
@@ -58,8 +58,9 @@ fn main() {
     assert_eq!(0b0000000000000000000000000010000000000000000000000000000000000000_0000000000000000000000000000000000001000000000000000000010000000u128.trailing_zeros(), 7);
     assert_eq!(core::intrinsics::saturating_sub(0, -170141183460469231731687303715884105728i128), 170141183460469231731687303715884105727i128);
 
-    let _d = 0i128.checked_div(2i128);
-    let _d = 0u128.checked_div(2u128);
+    std::hint::black_box(std::hint::black_box(7571400400375753350092698930310845914i128) * 10);
+    assert!(0i128.checked_div(2i128).is_some());
+    assert!(0u128.checked_div(2u128).is_some());
     assert_eq!(1u128 + 2, 3);
 
     assert_eq!(0b100010000000000000000000000000000u128 >> 10, 0b10001000000000000000000u128);
diff --git a/compiler/rustc_codegen_cranelift/patches/0003-rand-Disable-rand-tests-on-mingw.patch b/compiler/rustc_codegen_cranelift/patches/0003-rand-Disable-rand-tests-on-mingw.patch
index d8775e2d0..eb452c5cd 100644
--- a/compiler/rustc_codegen_cranelift/patches/0003-rand-Disable-rand-tests-on-mingw.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0003-rand-Disable-rand-tests-on-mingw.patch
@@ -19,8 +19,8 @@ index 217899e..9cedeb7 100644
 +    // This is broken on x86_64-pc-windows-gnu presumably due to a broken powf implementation
 +    #[cfg_attr(all(target_os = "windows", target_env = "gnu"), ignore)]
      fn value_stability() {
-         fn test_samples<F: Float + core::fmt::Debug, D: Distribution<F>>(
-             distr: D, zero: F, expected: &[F],
+         fn test_samples<F: Float + Debug + Display + LowerExp, D: Distribution<F>>(
+             distr: D, thresh: F, expected: &[F],
 diff --git a/rand_distr/tests/value_stability.rs b/rand_distr/tests/value_stability.rs
 index 192ba74..0101ace 100644
 --- a/rand_distr/tests/value_stability.rs
diff --git a/compiler/rustc_codegen_cranelift/patches/0022-sysroot-Disable-not-compiling-tests.patch b/compiler/rustc_codegen_cranelift/patches/0022-coretests-Disable-not-compiling-tests.patch
index 865aa833a..6afa5c71f 100644
--- a/compiler/rustc_codegen_cranelift/patches/0022-sysroot-Disable-not-compiling-tests.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0022-coretests-Disable-not-compiling-tests.patch
@@ -13,14 +13,14 @@ Subject: [PATCH] [core] Disable not compiling tests
  6 files changed, 16 insertions(+), 1 deletion(-)
  create mode 100644 library/core/tests/Cargo.toml
 
-diff --git a/library/core/tests/Cargo.toml b/library/core/tests/Cargo.toml
+diff --git a/Cargo.toml b/Cargo.toml
 new file mode 100644
 index 0000000..46fd999
 --- /dev/null
-+++ b/library/core/tests/Cargo.toml
++++ b/Cargo.toml
 @@ -0,0 +1,12 @@
 +[package]
-+name = "core"
++name = "coretests"
 +version = "0.0.0"
 +edition = "2021"
 +
@@ -31,5 +31,14 @@ index 0000000..46fd999
 +[dependencies]
 +rand = { version = "0.8.5", default-features = false }
 +rand_xorshift = { version = "0.3.0", default-features = false }
+diff --git a/lib.rs b/lib.rs
+index 42a26ae..5ac1042 100644
+--- a/lib.rs
++++ b/lib.rs
+@@ -1,3 +1,4 @@
++#![cfg(test)]
+ #![feature(alloc_layout_extra)]
+ #![feature(array_chunks)]
+ #![feature(array_methods)]
 --
 2.21.0 (Apple Git-122)
diff --git a/compiler/rustc_codegen_cranelift/patches/0023-sysroot-Ignore-failing-tests.patch b/compiler/rustc_codegen_cranelift/patches/0023-coretests-Ignore-failing-tests.patch
index f3cd7ee77..f2cb82751 100644
--- a/compiler/rustc_codegen_cranelift/patches/0023-sysroot-Ignore-failing-tests.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0023-coretests-Ignore-failing-tests.patch
@@ -10,10 +10,10 @@ Subject: [PATCH] [core] Ignore failing tests
  library/core/tests/time.rs       |  1 +
  4 files changed, 18 insertions(+), 2 deletions(-)
 
-diff --git a/library/core/tests/array.rs b/library/core/tests/array.rs
+diff --git a/array.rs b/array.rs
 index 4bc44e9..8e3c7a4 100644
---- a/library/core/tests/array.rs
-+++ b/library/core/tests/array.rs
+--- a/array.rs
++++ b/array.rs
 @@ -242,6 +242,7 @@ fn iterator_drops() {
      assert_eq!(i.get(), 5);
  }
@@ -46,10 +46,10 @@ index 4bc44e9..8e3c7a4 100644
  
  #[test]
  fn cell_allows_array_cycle() {
-diff --git a/library/core/tests/atomic.rs b/library/core/tests/atomic.rs
+diff --git a/atomic.rs b/atomic.rs
 index 13b12db..96fe4b9 100644
---- a/library/core/tests/atomic.rs
-+++ b/library/core/tests/atomic.rs
+--- a/atomic.rs
++++ b/atomic.rs
 @@ -185,6 +185,7 @@ fn ptr_bitops() {
  }
  
diff --git a/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch
new file mode 100644
index 000000000..1d5479bed
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch
@@ -0,0 +1,30 @@
+From ad7ffe71baba46865f2e65266ab025920dfdc20b Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Thu, 18 Feb 2021 18:45:28 +0100
+Subject: [PATCH] Disable 128bit atomic operations
+
+Cranelift doesn't support them yet
+---
+ library/core/src/panic/unwind_safe.rs |  6 -----
+ library/core/src/sync/atomic.rs       | 38 ---------------------------
+ library/core/tests/atomic.rs          |  4 ---
+ 4 files changed, 4 insertions(+), 50 deletions(-)
+
+diff --git a/atomic.rs b/atomic.rs
+index b735957..ea728b6 100644
+--- a/atomic.rs
++++ b/atomic.rs
+@@ -185,10 +185,6 @@ fn atomic_alignment() {
+     assert_eq!(align_of::<AtomicU64>(), size_of::<AtomicU64>());
+     #[cfg(target_has_atomic = "64")]
+     assert_eq!(align_of::<AtomicI64>(), size_of::<AtomicI64>());
+-    #[cfg(target_has_atomic = "128")]
+-    assert_eq!(align_of::<AtomicU128>(), size_of::<AtomicU128>());
+-    #[cfg(target_has_atomic = "128")]
+-    assert_eq!(align_of::<AtomicI128>(), size_of::<AtomicI128>());
+     #[cfg(target_has_atomic = "ptr")]
+     assert_eq!(align_of::<AtomicUsize>(), size_of::<AtomicUsize>());
+     #[cfg(target_has_atomic = "ptr")]
+--
+2.26.2.7.g19db9cfb68
+
diff --git a/compiler/rustc_codegen_cranelift/patches/0027-sysroot-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch
index 77f437974..45f73f36b 100644
--- a/compiler/rustc_codegen_cranelift/patches/0027-sysroot-128bit-atomic-operations.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch
@@ -85,21 +85,6 @@ index d9de37e..8293fce 100644
  
  macro_rules! atomic_int_ptr_sized {
      ( $($target_pointer_width:literal $align:literal)* ) => { $(
-diff --git a/library/core/tests/atomic.rs b/library/core/tests/atomic.rs
-index b735957..ea728b6 100644
---- a/library/core/tests/atomic.rs
-+++ b/library/core/tests/atomic.rs
-@@ -185,10 +185,6 @@ fn atomic_alignment() {
-     assert_eq!(align_of::<AtomicU64>(), size_of::<AtomicU64>());
-     #[cfg(target_has_atomic = "64")]
-     assert_eq!(align_of::<AtomicI64>(), size_of::<AtomicI64>());
--    #[cfg(target_has_atomic = "128")]
--    assert_eq!(align_of::<AtomicU128>(), size_of::<AtomicU128>());
--    #[cfg(target_has_atomic = "128")]
--    assert_eq!(align_of::<AtomicI128>(), size_of::<AtomicI128>());
-     #[cfg(target_has_atomic = "ptr")]
-     assert_eq!(align_of::<AtomicUsize>(), size_of::<AtomicUsize>());
-     #[cfg(target_has_atomic = "ptr")]
 --
 2.26.2.7.g19db9cfb68
 
diff --git a/compiler/rustc_codegen_cranelift/patches/0028-sysroot-Disable-long-running-tests.patch b/compiler/rustc_codegen_cranelift/patches/0028-coretests-Disable-long-running-tests.patch
index d804a78cc..440177018 100644
--- a/compiler/rustc_codegen_cranelift/patches/0028-sysroot-Disable-long-running-tests.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0028-coretests-Disable-long-running-tests.patch
@@ -7,10 +7,10 @@ Subject: [PATCH] Disable long running tests
  library/core/tests/slice.rs | 2 ++
  1 file changed, 2 insertions(+)
 
-diff --git a/library/core/tests/slice.rs b/library/core/tests/slice.rs
+diff --git a/slice.rs b/slice.rs
 index 8402833..84592e0 100644
---- a/library/core/tests/slice.rs
-+++ b/library/core/tests/slice.rs
+--- a/slice.rs
++++ b/slice.rs
 @@ -1809,6 +1809,7 @@ fn sort_unstable() {
      assert!(v == [0xDEADBEEF]);
  }
diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain
index 40fb54b91..2236a6ca1 100644
--- a/compiler/rustc_codegen_cranelift/rust-toolchain
+++ b/compiler/rustc_codegen_cranelift/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-02-06"
+channel = "nightly-2023-03-15"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs
index c993430b8..939a1f1ca 100644
--- a/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs
+++ b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs
@@ -1,11 +1,14 @@
 use std::env;
 #[cfg(unix)]
 use std::os::unix::process::CommandExt;
-use std::path::PathBuf;
 use std::process::Command;
 
 fn main() {
-    let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+    let current_exe = env::current_exe().unwrap();
+    let mut sysroot = current_exe.parent().unwrap();
+    if sysroot.file_name().unwrap().to_str().unwrap() == "bin" {
+        sysroot = sysroot.parent().unwrap();
+    }
 
     let mut rustflags = String::new();
     rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests -Zcodegen-backend=");
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs b/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs
index c187f54a6..b9bba7f2e 100644
--- a/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs
+++ b/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs
@@ -2,11 +2,14 @@ use std::env;
 use std::ffi::OsString;
 #[cfg(unix)]
 use std::os::unix::process::CommandExt;
-use std::path::PathBuf;
 use std::process::Command;
 
 fn main() {
-    let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+    let current_exe = env::current_exe().unwrap();
+    let mut sysroot = current_exe.parent().unwrap();
+    if sysroot.file_name().unwrap().to_str().unwrap() == "bin" {
+        sysroot = sysroot.parent().unwrap();
+    }
 
     let cg_clif_dylib_path = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join(
         env::consts::DLL_PREFIX.to_string() + "rustc_codegen_cranelift" + env::consts::DLL_SUFFIX,
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs b/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs
index a6528ac41..167631eaf 100644
--- a/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs
+++ b/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs
@@ -2,11 +2,14 @@ use std::env;
 use std::ffi::OsString;
 #[cfg(unix)]
 use std::os::unix::process::CommandExt;
-use std::path::PathBuf;
 use std::process::Command;
 
 fn main() {
-    let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
+    let current_exe = env::current_exe().unwrap();
+    let mut sysroot = current_exe.parent().unwrap();
+    if sysroot.file_name().unwrap().to_str().unwrap() == "bin" {
+        sysroot = sysroot.parent().unwrap();
+    }
 
     let cg_clif_dylib_path = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join(
         env::consts::DLL_PREFIX.to_string() + "rustc_codegen_cranelift" + env::consts::DLL_SUFFIX,
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustup.sh b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
index 34e3981b5..3cbeb6375 100755
--- a/compiler/rustc_codegen_cranelift/scripts/rustup.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
@@ -2,10 +2,24 @@
 
 set -e
 
+TOOLCHAIN=${TOOLCHAIN:-$(date +%Y-%m-%d)}
+
+function check_git_fixed_subtree() {
+    if [[ ! -e ./git-fixed-subtree.sh ]]; then
+        echo "Missing git-fixed-subtree.sh. Please run the following commands to download it:"
+        echo "curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/bjorn3/git/tqc-subtree-portable/contrib/subtree/git-subtree.sh -o git-fixed-subtree.sh"
+        echo "chmod u+x git-fixed-subtree.sh"
+        exit 1
+    fi
+    if [[ ! -x ./git-fixed-subtree.sh ]]; then
+        echo "git-fixed-subtree.sh is not executable. Please run the following command to make it executable:"
+        echo "chmod u+x git-fixed-subtree.sh"
+        exit 1
+    fi
+}
+
 case $1 in
     "prepare")
-        TOOLCHAIN=$(date +%Y-%m-%d)
-
         echo "=> Installing new nightly"
         rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists
         sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain
@@ -27,28 +41,35 @@ case $1 in
         git commit -m "Rustup to $(rustc -V)"
         ;;
     "push")
+        check_git_fixed_subtree
+
         cg_clif=$(pwd)
         pushd ../rust
         git pull origin master
         branch=sync_cg_clif-$(date +%Y-%m-%d)
         git checkout -b "$branch"
-        git subtree pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master
+        "$cg_clif/git-fixed-subtree.sh" pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master
         git push -u my "$branch"
 
         # immediately merge the merge commit into cg_clif to prevent merge conflicts when syncing
         # from rust-lang/rust later
-        git subtree push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
+        "$cg_clif/git-fixed-subtree.sh" push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
         popd
         git merge sync_from_rust
 	;;
     "pull")
+        check_git_fixed_subtree
+
+        RUST_VERS=$(curl "https://static.rust-lang.org/dist/$TOOLCHAIN/channel-rust-nightly-git-commit-hash.txt")
+        echo "Pulling $RUST_VERS ($TOOLCHAIN)"
+
         cg_clif=$(pwd)
         pushd ../rust
-        git pull origin master
-        rust_vers="$(git rev-parse HEAD)"
-        git subtree push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
+        git fetch origin master
+        git checkout "$RUST_VERS"
+        "$cg_clif/git-fixed-subtree.sh" push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust
         popd
-        git merge sync_from_rust -m "Sync from rust $rust_vers"
+        git merge sync_from_rust -m "Sync from rust $RUST_VERS"
         git branch -d sync_from_rust
         ;;
     *)
diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
index a08e80dd1..abb09775d 100644
--- a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
@@ -10,7 +10,7 @@ git fetch
 git checkout -- .
 git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
 
-git -c user.name=Dummy -c user.email=dummy@example.com am ../patches/*-sysroot-*.patch
+git -c user.name=Dummy -c user.email=dummy@example.com am ../patches/*-stdlib-*.patch
 
 git apply - <<EOF
 diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
@@ -36,7 +36,7 @@ changelog-seen = 2
 ninja = false
 
 [build]
-rustc = "$(pwd)/../dist/rustc-clif"
+rustc = "$(pwd)/../dist/bin/rustc-clif"
 cargo = "$(rustup which cargo)"
 full-bootstrap = true
 local-rebuild = true
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
index e14a129db..20dcb4cf3 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
@@ -21,6 +21,7 @@ done
 
 git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
 git checkout -- tests/ui/proc-macro/pretty-print-hack/
+rm tests/ui/parser/unclosed-delimiter-in-dep.rs # submodule contains //~ERROR
 
 # missing features
 # ================
@@ -80,6 +81,7 @@ rm tests/ui/layout/valid_range_oob.rs # different ICE message
 
 rm tests/ui/consts/issue-miri-1910.rs # different error message
 rm tests/ui/consts/offset_ub.rs # same
+rm tests/ui/consts/const-eval/ub-slice-get-unchecked.rs # same
 rm tests/ui/intrinsics/panic-uninitialized-zeroed.rs # same
 rm tests/ui/lint/lint-const-item-mutation.rs # same
 rm tests/ui/pattern/usefulness/doc-hidden-non-exhaustive.rs # same
@@ -110,12 +112,9 @@ rm tests/incremental/spike-neg2.rs # same
 
 rm tests/ui/simd/intrinsic/generic-reduction-pass.rs # simd_reduce_add_unordered doesn't accept an accumulator for integer vectors
 
-rm tests/ui/simd/intrinsic/generic-as.rs # crash when accessing vector type filed (#1318)
+rm tests/ui/simd/intrinsic/generic-as.rs # crash when accessing vector type field (#1318)
 rm tests/ui/simd/simd-bitmask.rs # crash
 
-rm tests/ui/dyn-star/dyn-star-to-dyn.rs
-rm tests/ui/dyn-star/dispatch-on-pin-mut.rs
-
 # bugs in the test suite
 # ======================
 rm tests/ui/backtrace.rs # TODO warning
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index 74396a66f..91c085d3d 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -4,6 +4,8 @@ mod comments;
 mod pass_mode;
 mod returning;
 
+use std::borrow::Cow;
+
 use cranelift_module::ModuleError;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::layout::FnAbiOf;
@@ -25,7 +27,7 @@ fn clif_sig_from_fn_abi<'tcx>(
 ) -> Signature {
     let call_conv = conv_to_call_conv(tcx.sess, fn_abi.conv, default_call_conv);
 
-    let inputs = fn_abi.args.iter().map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()).flatten();
+    let inputs = fn_abi.args.iter().flat_map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter());
 
     let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx);
     // Sometimes the first param is an pointer to the place where the return value needs to be stored.
@@ -116,7 +118,52 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         params: Vec<AbiParam>,
         returns: Vec<AbiParam>,
         args: &[Value],
-    ) -> &[Value] {
+    ) -> Cow<'_, [Value]> {
+        if self.tcx.sess.target.is_like_windows {
+            let (mut params, mut args): (Vec<_>, Vec<_>) =
+                params
+                    .into_iter()
+                    .zip(args)
+                    .map(|(param, &arg)| {
+                        if param.value_type == types::I128 {
+                            let arg_ptr = Pointer::stack_slot(self.bcx.create_sized_stack_slot(
+                                StackSlotData { kind: StackSlotKind::ExplicitSlot, size: 16 },
+                            ));
+                            arg_ptr.store(self, arg, MemFlags::trusted());
+                            (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self))
+                        } else {
+                            (param, arg)
+                        }
+                    })
+                    .unzip();
+
+            let indirect_ret_val = returns.len() == 1 && returns[0].value_type == types::I128;
+
+            if indirect_ret_val {
+                params.insert(0, AbiParam::new(self.pointer_type));
+                let ret_ptr =
+                    Pointer::stack_slot(self.bcx.create_sized_stack_slot(StackSlotData {
+                        kind: StackSlotKind::ExplicitSlot,
+                        size: 16,
+                    }));
+                args.insert(0, ret_ptr.get_addr(self));
+                self.lib_call_unadjusted(name, params, vec![], &args);
+                return Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]);
+            } else {
+                return self.lib_call_unadjusted(name, params, returns, &args);
+            }
+        }
+
+        self.lib_call_unadjusted(name, params, returns, args)
+    }
+
+    pub(crate) fn lib_call_unadjusted(
+        &mut self,
+        name: &str,
+        params: Vec<AbiParam>,
+        returns: Vec<AbiParam>,
+        args: &[Value],
+    ) -> Cow<'_, [Value]> {
         let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
         let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
         let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
@@ -125,41 +172,11 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         }
         let call_inst = self.bcx.ins().call(func_ref, args);
         if self.clif_comments.enabled() {
-            self.add_comment(call_inst, format!("easy_call {}", name));
+            self.add_comment(call_inst, format!("lib_call {}", name));
         }
         let results = self.bcx.inst_results(call_inst);
         assert!(results.len() <= 2, "{}", results.len());
-        results
-    }
-
-    pub(crate) fn easy_call(
-        &mut self,
-        name: &str,
-        args: &[CValue<'tcx>],
-        return_ty: Ty<'tcx>,
-    ) -> CValue<'tcx> {
-        let (input_tys, args): (Vec<_>, Vec<_>) = args
-            .iter()
-            .map(|arg| {
-                (AbiParam::new(self.clif_type(arg.layout().ty).unwrap()), arg.load_scalar(self))
-            })
-            .unzip();
-        let return_layout = self.layout_of(return_ty);
-        let return_tys = if let ty::Tuple(tup) = return_ty.kind() {
-            tup.iter().map(|ty| AbiParam::new(self.clif_type(ty).unwrap())).collect()
-        } else {
-            vec![AbiParam::new(self.clif_type(return_ty).unwrap())]
-        };
-        let ret_vals = self.lib_call(name, input_tys, return_tys, &args);
-        match *ret_vals {
-            [] => CValue::by_ref(
-                Pointer::const_addr(self, i64::from(self.pointer_type.bytes())),
-                return_layout,
-            ),
-            [val] => CValue::by_val(val, return_layout),
-            [val, extra] => CValue::by_val_pair(val, extra, return_layout),
-            _ => unreachable!(),
-        }
+        Cow::Borrowed(results)
     }
 }
 
@@ -275,10 +292,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
     self::comments::add_locals_header_comment(fx);
 
     for (local, arg_kind, ty) in func_params {
-        let layout = fx.layout_of(ty);
-
-        let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa;
-
         // While this is normally an optimization to prevent an unnecessary copy when an argument is
         // not mutated by the current function, this is necessary to support unsized arguments.
         if let ArgKind::Normal(Some(val)) = arg_kind {
@@ -300,6 +313,8 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
             }
         }
 
+        let layout = fx.layout_of(ty);
+        let is_ssa = ssa_analyzed[local].is_ssa(fx, ty);
         let place = make_local_place(fx, local, layout, is_ssa);
         assert_eq!(fx.local_map.push(place), local);
 
@@ -312,7 +327,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
             ArgKind::Spread(params) => {
                 for (i, param) in params.into_iter().enumerate() {
                     if let Some(param) = param {
-                        place.place_field(fx, mir::Field::new(i)).write_cvalue(fx, param);
+                        place.place_field(fx, FieldIdx::new(i)).write_cvalue(fx, param);
                     }
                 }
             }
@@ -323,7 +338,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
         let ty = fx.monomorphize(fx.mir.local_decls[local].ty);
         let layout = fx.layout_of(ty);
 
-        let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa;
+        let is_ssa = ssa_analyzed[local].is_ssa(fx, ty);
 
         let place = make_local_place(fx, local, layout, is_ssa);
         assert_eq!(fx.local_map.push(place), local);
@@ -445,7 +460,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         args.push(self_arg);
         for i in 0..tupled_arguments.len() {
             args.push(CallArgument {
-                value: pack_arg.value.value_field(fx, mir::Field::new(i)),
+                value: pack_arg.value.value_field(fx, FieldIdx::new(i)),
                 is_owned: pack_arg.is_owned,
             });
         }
@@ -515,10 +530,9 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                 args.into_iter()
                     .enumerate()
                     .skip(if first_arg_override.is_some() { 1 } else { 0 })
-                    .map(|(i, arg)| {
+                    .flat_map(|(i, arg)| {
                         adjust_arg_for_abi(fx, arg.value, &fn_abi.args[i], arg.is_owned).into_iter()
-                    })
-                    .flatten(),
+                    }),
             )
             .collect::<Vec<Value>>();
 
diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
index aaa141876..6d3e8eda2 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
@@ -14,7 +14,8 @@ pub(super) fn codegen_return_param<'tcx>(
 ) -> CPlace<'tcx> {
     let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode {
         PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => {
-            let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa;
+            let is_ssa =
+                ssa_analyzed[RETURN_PLACE].is_ssa(fx, fx.fn_abi.as_ref().unwrap().ret.layout.ty);
             (
                 super::make_local_place(
                     fx,
diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs
index 1c73957ca..2c246ceb3 100644
--- a/compiler/rustc_codegen_cranelift/src/allocator.rs
+++ b/compiler/rustc_codegen_cranelift/src/allocator.rs
@@ -4,6 +4,7 @@
 use crate::prelude::*;
 
 use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
+use rustc_codegen_ssa::base::allocator_kind_for_codegen;
 use rustc_session::config::OomStrategy;
 use rustc_span::symbol::sym;
 
@@ -13,24 +14,15 @@ pub(crate) fn codegen(
     module: &mut impl Module,
     unwind_context: &mut UnwindContext,
 ) -> bool {
-    let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| {
-        use rustc_middle::middle::dependency_format::Linkage;
-        list.iter().any(|&linkage| linkage == Linkage::Dynamic)
-    });
-    if any_dynamic_crate {
-        false
-    } else if let Some(kind) = tcx.allocator_kind(()) {
-        codegen_inner(
-            module,
-            unwind_context,
-            kind,
-            tcx.alloc_error_handler_kind(()).unwrap(),
-            tcx.sess.opts.unstable_opts.oom,
-        );
-        true
-    } else {
-        false
-    }
+    let Some(kind) = allocator_kind_for_codegen(tcx) else { return false };
+    codegen_inner(
+        module,
+        unwind_context,
+        kind,
+        tcx.alloc_error_handler_kind(()).unwrap(),
+        tcx.sess.opts.unstable_opts.oom,
+    );
+    true
 }
 
 fn codegen_inner(
diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs
index 0cbb9f3ec..54d5c1c2a 100644
--- a/compiler/rustc_codegen_cranelift/src/analyze.rs
+++ b/compiler/rustc_codegen_cranelift/src/analyze.rs
@@ -4,34 +4,30 @@ use crate::prelude::*;
 
 use rustc_index::vec::IndexVec;
 use rustc_middle::mir::StatementKind::*;
+use rustc_middle::ty::Ty;
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub(crate) enum SsaKind {
     NotSsa,
-    Ssa,
+    MaybeSsa,
+}
+
+impl SsaKind {
+    pub(crate) fn is_ssa<'tcx>(self, fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
+        self == SsaKind::MaybeSsa && (fx.clif_type(ty).is_some() || fx.clif_pair_type(ty).is_some())
+    }
 }
 
 pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> {
-    let mut flag_map = fx
-        .mir
-        .local_decls
-        .iter()
-        .map(|local_decl| {
-            let ty = fx.monomorphize(local_decl.ty);
-            if fx.clif_type(ty).is_some() || fx.clif_pair_type(ty).is_some() {
-                SsaKind::Ssa
-            } else {
-                SsaKind::NotSsa
-            }
-        })
-        .collect::<IndexVec<Local, SsaKind>>();
+    let mut flag_map =
+        fx.mir.local_decls.iter().map(|_| SsaKind::MaybeSsa).collect::<IndexVec<Local, SsaKind>>();
 
     for bb in fx.mir.basic_blocks.iter() {
         for stmt in bb.statements.iter() {
             match &stmt.kind {
                 Assign(place_and_rval) => match &place_and_rval.1 {
                     Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => {
-                        not_ssa(&mut flag_map, place.local)
+                        flag_map[place.local] = SsaKind::NotSsa;
                     }
                     _ => {}
                 },
@@ -42,7 +38,3 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> {
 
     flag_map
 }
-
-fn not_ssa(flag_map: &mut IndexVec<Local, SsaKind>, local: Local) {
-    flag_map[local] = SsaKind::NotSsa;
-}
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 7f857528c..f5301f9f7 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -192,7 +192,7 @@ pub(crate) fn compile_fn(
                         let pass_times = cranelift_codegen::timing::take_current();
                         // Replace newlines with | as measureme doesn't allow control characters like
                         // newlines inside strings.
-                        recorder.record_arg(format!("{}", pass_times).replace("\n", " | "));
+                        recorder.record_arg(format!("{}", pass_times).replace('\n', " | "));
                         recording_args = true;
                     },
                 )
@@ -345,18 +345,11 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
             TerminatorKind::Return => {
                 crate::abi::codegen_return(fx);
             }
-            TerminatorKind::Assert { cond, expected, msg, target, cleanup: _ } => {
-                if !fx.tcx.sess.overflow_checks() {
-                    let overflow_not_to_check = match msg {
-                        AssertKind::OverflowNeg(..) => true,
-                        AssertKind::Overflow(op, ..) => op.is_checkable(),
-                        _ => false,
-                    };
-                    if overflow_not_to_check {
-                        let target = fx.get_block(*target);
-                        fx.bcx.ins().jump(target, &[]);
-                        continue;
-                    }
+            TerminatorKind::Assert { cond, expected, msg, target, unwind: _ } => {
+                if !fx.tcx.sess.overflow_checks() && msg.is_optional_overflow_check() {
+                    let target = fx.get_block(*target);
+                    fx.bcx.ins().jump(target, &[]);
+                    continue;
                 }
                 let cond = codegen_operand(fx, cond).load_scalar(fx);
 
@@ -365,11 +358,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 fx.bcx.set_cold_block(failure);
 
                 if *expected {
-                    fx.bcx.ins().brz(cond, failure, &[]);
+                    fx.bcx.ins().brif(cond, target, &[], failure, &[]);
                 } else {
-                    fx.bcx.ins().brnz(cond, failure, &[]);
+                    fx.bcx.ins().brif(cond, failure, &[], target, &[]);
                 };
-                fx.bcx.ins().jump(target, &[]);
 
                 fx.bcx.switch_to_block(failure);
                 fx.bcx.ins().nop();
@@ -387,6 +379,18 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                             source_info.span,
                         );
                     }
+                    AssertKind::MisalignedPointerDereference { ref required, ref found } => {
+                        let required = codegen_operand(fx, required).load_scalar(fx);
+                        let found = codegen_operand(fx, found).load_scalar(fx);
+                        let location = fx.get_caller_location(source_info).load_scalar(fx);
+
+                        codegen_panic_inner(
+                            fx,
+                            rustc_hir::LangItem::PanicBoundsCheck,
+                            &[required, found, location],
+                            source_info.span,
+                        );
+                    }
                     _ => {
                         let msg_str = msg.description();
                         codegen_panic(fx, msg_str, source_info);
@@ -425,11 +429,9 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                         }
                     } else {
                         if test_zero {
-                            fx.bcx.ins().brz(discr, then_block, &[]);
-                            fx.bcx.ins().jump(else_block, &[]);
+                            fx.bcx.ins().brif(discr, else_block, &[], then_block, &[]);
                         } else {
-                            fx.bcx.ins().brnz(discr, then_block, &[]);
-                            fx.bcx.ins().jump(else_block, &[]);
+                            fx.bcx.ins().brif(discr, then_block, &[], else_block, &[]);
                         }
                     }
                 } else {
@@ -448,7 +450,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 destination,
                 target,
                 fn_span,
-                cleanup: _,
+                unwind: _,
                 from_hir_call: _,
             } => {
                 fx.tcx.prof.generic_activity("codegen call").run(|| {
@@ -468,7 +470,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 options,
                 destination,
                 line_spans: _,
-                cleanup: _,
+                unwind: _,
             } => {
                 if options.contains(InlineAsmOptions::MAY_UNWIND) {
                     fx.tcx.sess.span_fatal(
@@ -486,7 +488,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     *destination,
                 );
             }
-            TerminatorKind::Abort => {
+            TerminatorKind::Terminate => {
                 codegen_panic_cannot_unwind(fx, source_info);
             }
             TerminatorKind::Resume => {
@@ -499,7 +501,6 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
             TerminatorKind::Yield { .. }
             | TerminatorKind::FalseEdge { .. }
             | TerminatorKind::FalseUnwind { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::GeneratorDrop => {
                 bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
             }
@@ -720,6 +721,10 @@ fn codegen_stmt<'tcx>(
                     let operand = codegen_operand(fx, operand);
                     operand.coerce_dyn_star(fx, lval);
                 }
+                Rvalue::Cast(CastKind::Transmute, ref operand, _to_ty) => {
+                    let operand = codegen_operand(fx, operand);
+                    lval.write_cvalue_transmute(fx, operand);
+                }
                 Rvalue::Discriminant(place) => {
                     let place = codegen_place(fx, place);
                     let value = place.to_cvalue(fx);
@@ -751,8 +756,7 @@ fn codegen_stmt<'tcx>(
 
                         fx.bcx.switch_to_block(loop_block);
                         let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64);
-                        fx.bcx.ins().brnz(done, done_block, &[]);
-                        fx.bcx.ins().jump(loop_block2, &[]);
+                        fx.bcx.ins().brif(done, done_block, &[], loop_block2, &[]);
 
                         fx.bcx.switch_to_block(loop_block2);
                         let to = lval.place_index(fx, index);
@@ -793,19 +797,20 @@ fn codegen_stmt<'tcx>(
                             let variant_dest = lval.downcast_variant(fx, variant_index);
                             (variant_index, variant_dest, active_field_index)
                         }
-                        _ => (VariantIdx::from_u32(0), lval, None),
+                        _ => (FIRST_VARIANT, lval, None),
                     };
                     if active_field_index.is_some() {
                         assert_eq!(operands.len(), 1);
                     }
-                    for (i, operand) in operands.iter().enumerate() {
+                    for (i, operand) in operands.iter_enumerated() {
                         let operand = codegen_operand(fx, operand);
                         let field_index = active_field_index.unwrap_or(i);
                         let to = if let mir::AggregateKind::Array(_) = **kind {
-                            let index = fx.bcx.ins().iconst(fx.pointer_type, field_index as i64);
+                            let array_index = i64::from(field_index.as_u32());
+                            let index = fx.bcx.ins().iconst(fx.pointer_type, array_index);
                             variant_dest.place_index(fx, index)
                         } else {
-                            variant_dest.place_field(fx, mir::Field::new(field_index))
+                            variant_dest.place_field(fx, field_index)
                         };
                         to.write_cvalue(fx, operand);
                     }
@@ -820,6 +825,7 @@ fn codegen_stmt<'tcx>(
         | StatementKind::Nop
         | StatementKind::FakeRead(..)
         | StatementKind::Retag { .. }
+        | StatementKind::PlaceMention(..)
         | StatementKind::AscribeUserType(..) => {}
 
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
@@ -997,7 +1003,7 @@ fn codegen_panic_inner<'tcx>(
     let symbol_name = fx.tcx.symbol_name(instance).name;
 
     fx.lib_call(
-        &*symbol_name,
+        symbol_name,
         args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(),
         vec![],
         args,
diff --git a/compiler/rustc_codegen_cranelift/src/cast.rs b/compiler/rustc_codegen_cranelift/src/cast.rs
index 5091c5a9f..032d11510 100644
--- a/compiler/rustc_codegen_cranelift/src/cast.rs
+++ b/compiler/rustc_codegen_cranelift/src/cast.rs
@@ -64,17 +64,12 @@ pub(crate) fn clif_int_or_float_cast(
                 },
             );
 
-            let from_rust_ty = if from_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-
-            let to_rust_ty = match to_ty {
-                types::F32 => fx.tcx.types.f32,
-                types::F64 => fx.tcx.types.f64,
-                _ => unreachable!(),
-            };
-
-            return fx
-                .easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
-                .load_scalar(fx);
+            return fx.lib_call(
+                &name,
+                vec![AbiParam::new(types::I128)],
+                vec![AbiParam::new(to_ty)],
+                &[from],
+            )[0];
         }
 
         // int-like -> float
@@ -101,16 +96,29 @@ pub(crate) fn clif_int_or_float_cast(
                 },
             );
 
-            let from_rust_ty = match from_ty {
-                types::F32 => fx.tcx.types.f32,
-                types::F64 => fx.tcx.types.f64,
-                _ => unreachable!(),
-            };
-
-            let to_rust_ty = if to_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-
-            fx.easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
-                .load_scalar(fx)
+            if fx.tcx.sess.target.is_like_windows {
+                let ret = fx.lib_call(
+                    &name,
+                    vec![AbiParam::new(from_ty)],
+                    vec![AbiParam::new(types::I64X2)],
+                    &[from],
+                )[0];
+                // FIXME use bitcast instead of store to get from i64x2 to i128
+                let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
+                    kind: StackSlotKind::ExplicitSlot,
+                    size: 16,
+                });
+                let ret_ptr = Pointer::stack_slot(stack_slot);
+                ret_ptr.store(fx, ret, MemFlags::trusted());
+                ret_ptr.load(fx, types::I128, MemFlags::trusted())
+            } else {
+                fx.lib_call(
+                    &name,
+                    vec![AbiParam::new(from_ty)],
+                    vec![AbiParam::new(types::I128)],
+                    &[from],
+                )[0]
+            }
         } else if to_ty == types::I8 || to_ty == types::I16 {
             // FIXME implement fcvt_to_*int_sat.i8/i16
             let val = if to_signed {
diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
index 40bfe7077..f674ce776 100644
--- a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
+++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
@@ -29,39 +29,24 @@ pub(crate) fn maybe_codegen<'tcx>(
         BinOp::Add | BinOp::Sub if !checked => None,
         BinOp::Mul if !checked || is_signed => {
             if !checked {
-                let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-                if fx.tcx.sess.target.is_like_windows {
-                    let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
-                    let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
-                    let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
-                    assert!(lhs_extra.is_none());
-                    assert!(rhs_extra.is_none());
-                    let args = [
-                        ret_place.to_ptr().get_addr(fx),
-                        lhs_ptr.get_addr(fx),
-                        rhs_ptr.get_addr(fx),
-                    ];
-                    fx.lib_call(
-                        "__multi3",
-                        vec![
-                            AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn),
-                            AbiParam::new(fx.pointer_type),
-                            AbiParam::new(fx.pointer_type),
-                        ],
-                        vec![],
-                        &args,
-                    );
-                    Some(ret_place.to_cvalue(fx))
-                } else {
-                    Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
-                }
+                let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)];
+                let ret_val = fx.lib_call(
+                    "__multi3",
+                    vec![AbiParam::new(types::I128), AbiParam::new(types::I128)],
+                    vec![AbiParam::new(types::I128)],
+                    &args,
+                )[0];
+                Some(CValue::by_val(
+                    ret_val,
+                    fx.layout_of(if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 }),
+                ))
             } else {
                 let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]);
                 let oflow = CPlace::new_stack_slot(fx, fx.layout_of(fx.tcx.types.i32));
                 let lhs = lhs.load_scalar(fx);
                 let rhs = rhs.load_scalar(fx);
                 let oflow_ptr = oflow.to_ptr().get_addr(fx);
-                let res = fx.lib_call(
+                let res = fx.lib_call_unadjusted(
                     "__muloti4",
                     vec![
                         AbiParam::new(types::I128),
@@ -80,29 +65,12 @@ pub(crate) fn maybe_codegen<'tcx>(
             assert!(checked);
             let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]);
             let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
-            let (param_types, args) = if fx.tcx.sess.target.is_like_windows {
-                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
-                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
-                assert!(lhs_extra.is_none());
-                assert!(rhs_extra.is_none());
-                (
-                    vec![
-                        AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn),
-                        AbiParam::new(fx.pointer_type),
-                        AbiParam::new(fx.pointer_type),
-                    ],
-                    [out_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)],
-                )
-            } else {
-                (
-                    vec![
-                        AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn),
-                        AbiParam::new(types::I128),
-                        AbiParam::new(types::I128),
-                    ],
-                    [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)],
-                )
-            };
+            let param_types = vec![
+                AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn),
+                AbiParam::new(types::I128),
+                AbiParam::new(types::I128),
+            ];
+            let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
             let name = match (bin_op, is_signed) {
                 (BinOp::Add, false) => "__rust_u128_addo",
                 (BinOp::Add, true) => "__rust_i128_addo",
@@ -125,14 +93,10 @@ pub(crate) fn maybe_codegen<'tcx>(
                 _ => unreachable!(),
             };
             if fx.tcx.sess.target.is_like_windows {
-                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
-                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
-                assert!(lhs_extra.is_none());
-                assert!(rhs_extra.is_none());
-                let args = [lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)];
                 let ret = fx.lib_call(
                     name,
-                    vec![AbiParam::new(fx.pointer_type), AbiParam::new(fx.pointer_type)],
+                    vec![AbiParam::new(types::I128), AbiParam::new(types::I128)],
                     vec![AbiParam::new(types::I64X2)],
                     &args,
                 )[0];
@@ -141,7 +105,14 @@ pub(crate) fn maybe_codegen<'tcx>(
                 ret_place.to_ptr().store(fx, ret, MemFlags::trusted());
                 Some(ret_place.to_cvalue(fx))
             } else {
-                Some(fx.easy_call(name, &[lhs, rhs], lhs.layout().ty))
+                let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)];
+                let ret_val = fx.lib_call(
+                    name,
+                    vec![AbiParam::new(types::I128), AbiParam::new(types::I128)],
+                    vec![AbiParam::new(types::I128)],
+                    &args,
+                )[0];
+                Some(CValue::by_val(ret_val, lhs.layout()))
             }
         }
         BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => {
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index 722e2754e..d39bf7000 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -75,7 +75,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
         ty::Adt(adt_def, _) if adt_def.repr().simd() => {
             let (element, count) = match &tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().abi
             {
-                Abi::Vector { element, count } => (element.clone(), *count),
+                Abi::Vector { element, count } => (*element, *count),
                 _ => unreachable!(),
             };
 
diff --git a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
index 8a53baa76..f3b963200 100644
--- a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
+++ b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
@@ -39,6 +39,7 @@ builtin_functions! {
 
     // integers
     fn __multi3(a: i128, b: i128) -> i128;
+    fn __muloti4(n: i128, d: i128, oflow: &mut i32) -> i128;
     fn __udivti3(n: u128, d: u128) -> u128;
     fn __divti3(n: i128, d: i128) -> i128;
     fn __umodti3(n: u128, d: u128) -> u128;
diff --git a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
index f855e20e0..203219a8a 100644
--- a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
+++ b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
@@ -32,7 +32,7 @@ impl ConcurrencyLimiter {
         ConcurrencyLimiter {
             helper_thread: Some(helper_thread),
             state,
-            available_token_condvar: Arc::new(Condvar::new()),
+            available_token_condvar,
             finished: false,
         }
     }
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index 49c4f1aaa..e87f4e258 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -54,12 +54,22 @@ pub(crate) fn codegen_tls_ref<'tcx>(
     def_id: DefId,
     layout: TyAndLayout<'tcx>,
 ) -> CValue<'tcx> {
-    let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
-    let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    if fx.clif_comments.enabled() {
-        fx.add_comment(local_data_id, format!("tls {:?}", def_id));
-    }
-    let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id);
+    let tls_ptr = if !def_id.is_local() && fx.tcx.needs_thread_local_shim(def_id) {
+        let instance = ty::Instance {
+            def: ty::InstanceDef::ThreadLocalShim(def_id),
+            substs: ty::InternalSubsts::empty(),
+        };
+        let func_ref = fx.get_function_ref(instance);
+        let call = fx.bcx.ins().call(func_ref, &[]);
+        fx.bcx.func.dfg.first_result(call)
+    } else {
+        let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
+        let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+        if fx.clif_comments.enabled() {
+            fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+        }
+        fx.bcx.ins().tls_value(fx.pointer_type, local_data_id)
+    };
     CValue::by_val(tls_ptr, layout)
 }
 
@@ -290,7 +300,7 @@ fn data_id_for_static(
         };
 
         let data_id = match module.declare_data(
-            &*symbol_name,
+            symbol_name,
             linkage,
             is_mutable,
             attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL),
@@ -338,7 +348,7 @@ fn data_id_for_static(
     };
 
     let data_id = match module.declare_data(
-        &*symbol_name,
+        symbol_name,
         linkage,
         is_mutable,
         attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL),
@@ -529,6 +539,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                         | StatementKind::StorageDead(_)
                         | StatementKind::Retag(_, _)
                         | StatementKind::AscribeUserType(_, _)
+                        | StatementKind::PlaceMention(..)
                         | StatementKind::Coverage(_)
                         | StatementKind::ConstEvalCounter
                         | StatementKind::Nop => {}
@@ -538,13 +549,12 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                     TerminatorKind::Goto { .. }
                     | TerminatorKind::SwitchInt { .. }
                     | TerminatorKind::Resume
-                    | TerminatorKind::Abort
+                    | TerminatorKind::Terminate
                     | TerminatorKind::Return
                     | TerminatorKind::Unreachable
                     | TerminatorKind::Drop { .. }
                     | TerminatorKind::Assert { .. } => {}
-                    TerminatorKind::DropAndReplace { .. }
-                    | TerminatorKind::Yield { .. }
+                    TerminatorKind::Yield { .. }
                     | TerminatorKind::GeneratorDrop
                     | TerminatorKind::FalseEdge { .. }
                     | TerminatorKind::FalseUnwind { .. } => unreachable!(),
diff --git a/compiler/rustc_codegen_cranelift/src/cranelift_native.rs b/compiler/rustc_codegen_cranelift/src/cranelift_native.rs
deleted file mode 100644
index 6c4efca44..000000000
--- a/compiler/rustc_codegen_cranelift/src/cranelift_native.rs
+++ /dev/null
@@ -1,248 +0,0 @@
-// Vendored from https://github.com/bytecodealliance/wasmtime/blob/b58a197d33f044193c3d608010f5e6ec394ac07e/cranelift/native/src/lib.rs
-// which is licensed as
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-// unlike rustc_codegen_cranelift itself. Also applies a small change to remove #![cfg_attr] that
-// rust's CI complains about and to fix formatting to match rustc.
-// FIXME revert back to the external crate with Cranelift 0.93
-#![allow(warnings)]
-
-//! Performs autodetection of the host for the purposes of running
-//! Cranelift to generate code to run on the same machine.
-
-#![deny(missing_docs, trivial_numeric_casts, unused_extern_crates, unstable_features)]
-#![warn(unused_import_braces)]
-
-use cranelift_codegen::isa;
-use target_lexicon::Triple;
-
-/// Return an `isa` builder configured for the current host
-/// machine, or `Err(())` if the host machine is not supported
-/// in the current configuration.
-pub fn builder() -> Result<isa::Builder, &'static str> {
-    builder_with_options(true)
-}
-
-/// Return an `isa` builder configured for the current host
-/// machine, or `Err(())` if the host machine is not supported
-/// in the current configuration.
-///
-/// Selects the given backend variant specifically; this is
-/// useful when more than oen backend exists for a given target
-/// (e.g., on x86-64).
-pub fn builder_with_options(infer_native_flags: bool) -> Result<isa::Builder, &'static str> {
-    let mut isa_builder = isa::lookup(Triple::host()).map_err(|err| match err {
-        isa::LookupError::SupportDisabled => "support for architecture disabled at compile time",
-        isa::LookupError::Unsupported => "unsupported architecture",
-    })?;
-
-    #[cfg(target_arch = "x86_64")]
-    {
-        use cranelift_codegen::settings::Configurable;
-
-        if !std::is_x86_feature_detected!("sse2") {
-            return Err("x86 support requires SSE2");
-        }
-
-        if !infer_native_flags {
-            return Ok(isa_builder);
-        }
-
-        // These are temporarily enabled by default (see #3810 for
-        // more) so that a default-constructed `Flags` can work with
-        // default Wasmtime features. Otherwise, the user must
-        // explicitly use native flags or turn these on when on x86-64
-        // platforms to avoid a configuration panic. In order for the
-        // "enable if detected" logic below to work, we must turn them
-        // *off* (differing from the default) and then re-enable below
-        // if present.
-        isa_builder.set("has_sse3", "false").unwrap();
-        isa_builder.set("has_ssse3", "false").unwrap();
-        isa_builder.set("has_sse41", "false").unwrap();
-        isa_builder.set("has_sse42", "false").unwrap();
-
-        if std::is_x86_feature_detected!("sse3") {
-            isa_builder.enable("has_sse3").unwrap();
-        }
-        if std::is_x86_feature_detected!("ssse3") {
-            isa_builder.enable("has_ssse3").unwrap();
-        }
-        if std::is_x86_feature_detected!("sse4.1") {
-            isa_builder.enable("has_sse41").unwrap();
-        }
-        if std::is_x86_feature_detected!("sse4.2") {
-            isa_builder.enable("has_sse42").unwrap();
-        }
-        if std::is_x86_feature_detected!("popcnt") {
-            isa_builder.enable("has_popcnt").unwrap();
-        }
-        if std::is_x86_feature_detected!("avx") {
-            isa_builder.enable("has_avx").unwrap();
-        }
-        if std::is_x86_feature_detected!("avx2") {
-            isa_builder.enable("has_avx2").unwrap();
-        }
-        if std::is_x86_feature_detected!("fma") {
-            isa_builder.enable("has_fma").unwrap();
-        }
-        if std::is_x86_feature_detected!("bmi1") {
-            isa_builder.enable("has_bmi1").unwrap();
-        }
-        if std::is_x86_feature_detected!("bmi2") {
-            isa_builder.enable("has_bmi2").unwrap();
-        }
-        if std::is_x86_feature_detected!("avx512bitalg") {
-            isa_builder.enable("has_avx512bitalg").unwrap();
-        }
-        if std::is_x86_feature_detected!("avx512dq") {
-            isa_builder.enable("has_avx512dq").unwrap();
-        }
-        if std::is_x86_feature_detected!("avx512f") {
-            isa_builder.enable("has_avx512f").unwrap();
-        }
-        if std::is_x86_feature_detected!("avx512vl") {
-            isa_builder.enable("has_avx512vl").unwrap();
-        }
-        if std::is_x86_feature_detected!("avx512vbmi") {
-            isa_builder.enable("has_avx512vbmi").unwrap();
-        }
-        if std::is_x86_feature_detected!("lzcnt") {
-            isa_builder.enable("has_lzcnt").unwrap();
-        }
-    }
-
-    #[cfg(target_arch = "aarch64")]
-    {
-        use cranelift_codegen::settings::Configurable;
-
-        if !infer_native_flags {
-            return Ok(isa_builder);
-        }
-
-        if std::arch::is_aarch64_feature_detected!("lse") {
-            isa_builder.enable("has_lse").unwrap();
-        }
-
-        if std::arch::is_aarch64_feature_detected!("paca") {
-            isa_builder.enable("has_pauth").unwrap();
-        }
-
-        if cfg!(target_os = "macos") {
-            // Pointer authentication is always available on Apple Silicon.
-            isa_builder.enable("sign_return_address").unwrap();
-            // macOS enforces the use of the B key for return addresses.
-            isa_builder.enable("sign_return_address_with_bkey").unwrap();
-        }
-    }
-
-    // There is no is_s390x_feature_detected macro yet, so for now
-    // we use getauxval from the libc crate directly.
-    #[cfg(all(target_arch = "s390x", target_os = "linux"))]
-    {
-        use cranelift_codegen::settings::Configurable;
-
-        if !infer_native_flags {
-            return Ok(isa_builder);
-        }
-
-        let v = unsafe { libc::getauxval(libc::AT_HWCAP) };
-        const HWCAP_S390X_VXRS_EXT2: libc::c_ulong = 32768;
-        if (v & HWCAP_S390X_VXRS_EXT2) != 0 {
-            isa_builder.enable("has_vxrs_ext2").unwrap();
-            // There is no separate HWCAP bit for mie2, so assume
-            // that any machine with vxrs_ext2 also has mie2.
-            isa_builder.enable("has_mie2").unwrap();
-        }
-    }
-
-    // `is_riscv_feature_detected` is nightly only for now, use
-    // getauxval from the libc crate directly as a temporary measure.
-    #[cfg(all(target_arch = "riscv64", target_os = "linux"))]
-    {
-        use cranelift_codegen::settings::Configurable;
-
-        if !infer_native_flags {
-            return Ok(isa_builder);
-        }
-
-        let v = unsafe { libc::getauxval(libc::AT_HWCAP) };
-
-        const HWCAP_RISCV_EXT_A: libc::c_ulong = 1 << (b'a' - b'a');
-        const HWCAP_RISCV_EXT_C: libc::c_ulong = 1 << (b'c' - b'a');
-        const HWCAP_RISCV_EXT_D: libc::c_ulong = 1 << (b'd' - b'a');
-        const HWCAP_RISCV_EXT_F: libc::c_ulong = 1 << (b'f' - b'a');
-        const HWCAP_RISCV_EXT_M: libc::c_ulong = 1 << (b'm' - b'a');
-        const HWCAP_RISCV_EXT_V: libc::c_ulong = 1 << (b'v' - b'a');
-
-        if (v & HWCAP_RISCV_EXT_A) != 0 {
-            isa_builder.enable("has_a").unwrap();
-        }
-
-        if (v & HWCAP_RISCV_EXT_C) != 0 {
-            isa_builder.enable("has_c").unwrap();
-        }
-
-        if (v & HWCAP_RISCV_EXT_D) != 0 {
-            isa_builder.enable("has_d").unwrap();
-        }
-
-        if (v & HWCAP_RISCV_EXT_F) != 0 {
-            isa_builder.enable("has_f").unwrap();
-
-            // TODO: There doesn't seem to be a bit associated with this extension
-            // rust enables it with the `f` extension:
-            // https://github.com/rust-lang/stdarch/blob/790411f93c4b5eada3c23abb4c9a063fb0b24d99/crates/std_detect/src/detect/os/linux/riscv.rs#L43
-            isa_builder.enable("has_zicsr").unwrap();
-        }
-
-        if (v & HWCAP_RISCV_EXT_M) != 0 {
-            isa_builder.enable("has_m").unwrap();
-        }
-
-        if (v & HWCAP_RISCV_EXT_V) != 0 {
-            isa_builder.enable("has_v").unwrap();
-        }
-
-        // TODO: ZiFencei does not have a bit associated with it
-        // TODO: Zbkb does not have a bit associated with it
-    }
-
-    // squelch warnings about unused mut/variables on some platforms.
-    drop(&mut isa_builder);
-    drop(infer_native_flags);
-
-    Ok(isa_builder)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::builder;
-    use cranelift_codegen::isa::CallConv;
-    use cranelift_codegen::settings;
-
-    #[test]
-    fn test() {
-        if let Ok(isa_builder) = builder() {
-            let flag_builder = settings::builder();
-            let isa = isa_builder.finish(settings::Flags::new(flag_builder)).unwrap();
-
-            if cfg!(all(target_os = "macos", target_arch = "aarch64")) {
-                assert_eq!(isa.default_call_conv(), CallConv::AppleAarch64);
-            } else if cfg!(any(unix, target_os = "nebulet")) {
-                assert_eq!(isa.default_call_conv(), CallConv::SystemV);
-            } else if cfg!(windows) {
-                assert_eq!(isa.default_call_conv(), CallConv::WindowsFastcall);
-            }
-
-            if cfg!(target_pointer_width = "64") {
-                assert_eq!(isa.pointer_bits(), 64);
-            } else if cfg!(target_pointer_width = "32") {
-                assert_eq!(isa.pointer_bits(), 32);
-            } else if cfg!(target_pointer_width = "16") {
-                assert_eq!(isa.pointer_bits(), 16);
-            }
-        }
-    }
-}
-
-/// Version number of this crate.
-pub const VERSION: &str = env!("CARGO_PKG_VERSION");
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
index 9583cd2ec..c4a5627e6 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
@@ -113,7 +113,7 @@ impl Writer for WriterRelocate {
                     offset: offset as u32,
                     size,
                     name: DebugRelocName::Symbol(symbol),
-                    addend: addend as i64,
+                    addend,
                     kind: object::RelocationKind::Absolute,
                 });
                 self.write_udata(0, size)
diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs
index 3cbf313ad..670384663 100644
--- a/compiler/rustc_codegen_cranelift/src/discriminant.rs
+++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs
@@ -26,7 +26,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
             tag_encoding: TagEncoding::Direct,
             variants: _,
         } => {
-            let ptr = place.place_field(fx, mir::Field::new(tag_field));
+            let ptr = place.place_field(fx, FieldIdx::new(tag_field));
             let to = layout.ty.discriminant_for_variant(fx.tcx, variant_index).unwrap().val;
             let to = if ptr.layout().abi.is_signed() {
                 ty::ScalarInt::try_from_int(
@@ -47,7 +47,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
             variants: _,
         } => {
             if variant_index != untagged_variant {
-                let niche = place.place_field(fx, mir::Field::new(tag_field));
+                let niche = place.place_field(fx, FieldIdx::new(tag_field));
                 let niche_type = fx.clif_type(niche.layout().ty).unwrap();
                 let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
                 let niche_value = (niche_value as u128).wrapping_add(niche_start);
@@ -103,11 +103,10 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
         }
     };
 
-    let cast_to_size = dest_layout.layout.size();
     let cast_to = fx.clif_type(dest_layout.ty).unwrap();
 
     // Read the tag/niche-encoded discriminant from memory.
-    let tag = value.value_field(fx, mir::Field::new(tag_field));
+    let tag = value.value_field(fx, FieldIdx::new(tag_field));
     let tag = tag.load_scalar(fx);
 
     // Decode the discriminant (specifically if it's niche-encoded).
@@ -122,21 +121,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
             dest.write_cvalue(fx, res);
         }
         TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
-            let tag_size = tag_scalar.size(fx);
-            let max_unsigned = tag_size.unsigned_int_max();
-            let max_signed = tag_size.signed_int_max() as u128;
-            let min_signed = max_signed + 1;
             let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
-            let niche_end = niche_start.wrapping_add(relative_max as u128) & max_unsigned;
-            let range = tag_scalar.valid_range(fx);
-
-            let sle = |lhs: u128, rhs: u128| -> bool {
-                // Signed and unsigned comparisons give the same results,
-                // except that in signed comparisons an integer with the
-                // sign bit set is less than one with the sign bit clear.
-                // Toggle the sign bit to do a signed comparison.
-                (lhs ^ min_signed) <= (rhs ^ min_signed)
-            };
 
             // We have a subrange `niche_start..=niche_end` inside `range`.
             // If the value of the tag is inside this subrange, it's a
@@ -153,45 +138,6 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
             // }
             // However, we will likely be able to emit simpler code.
 
-            // Find the least and greatest values in `range`, considered
-            // both as signed and unsigned.
-            let (low_unsigned, high_unsigned) =
-                if range.start <= range.end { (range.start, range.end) } else { (0, max_unsigned) };
-            let (low_signed, high_signed) = if sle(range.start, range.end) {
-                (range.start, range.end)
-            } else {
-                (min_signed, max_signed)
-            };
-
-            let niches_ule = niche_start <= niche_end;
-            let niches_sle = sle(niche_start, niche_end);
-            let cast_smaller = cast_to_size <= tag_size;
-
-            // In the algorithm above, we can change
-            // cast(relative_tag) + niche_variants.start()
-            // into
-            // cast(tag + (niche_variants.start() - niche_start))
-            // if either the casted type is no larger than the original
-            // type, or if the niche values are contiguous (in either the
-            // signed or unsigned sense).
-            let can_incr = cast_smaller || niches_ule || niches_sle;
-
-            let data_for_boundary_niche = || -> Option<(IntCC, u128)> {
-                if !can_incr {
-                    None
-                } else if niche_start == low_unsigned {
-                    Some((IntCC::UnsignedLessThanOrEqual, niche_end))
-                } else if niche_end == high_unsigned {
-                    Some((IntCC::UnsignedGreaterThanOrEqual, niche_start))
-                } else if niche_start == low_signed {
-                    Some((IntCC::SignedLessThanOrEqual, niche_end))
-                } else if niche_end == high_signed {
-                    Some((IntCC::SignedGreaterThanOrEqual, niche_start))
-                } else {
-                    None
-                }
-            };
-
             let (is_niche, tagged_discr, delta) = if relative_max == 0 {
                 // Best case scenario: only one tagged variant. This will
                 // likely become just a comparison and a jump.
@@ -206,41 +152,6 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
                 let tagged_discr =
                     fx.bcx.ins().iconst(cast_to, niche_variants.start().as_u32() as i64);
                 (is_niche, tagged_discr, 0)
-            } else if let Some((predicate, constant)) = data_for_boundary_niche() {
-                // The niche values are either the lowest or the highest in
-                // `range`. We can avoid the first subtraction in the
-                // algorithm.
-                // The algorithm is now this:
-                // is_niche = tag <= niche_end
-                // discr = if is_niche {
-                //     cast(tag + (niche_variants.start() - niche_start))
-                // } else {
-                //     untagged_variant
-                // }
-                // (the first line may instead be tag >= niche_start,
-                // and may be a signed or unsigned comparison)
-                // The arithmetic must be done before the cast, so we can
-                // have the correct wrapping behavior. See issue #104519 for
-                // the consequences of getting this wrong.
-                let is_niche = codegen_icmp_imm(fx, predicate, tag, constant as i128);
-                let delta = (niche_variants.start().as_u32() as u128).wrapping_sub(niche_start);
-                let incr_tag = if delta == 0 {
-                    tag
-                } else {
-                    let delta = match fx.bcx.func.dfg.value_type(tag) {
-                        types::I128 => {
-                            let lsb = fx.bcx.ins().iconst(types::I64, delta as u64 as i64);
-                            let msb = fx.bcx.ins().iconst(types::I64, (delta >> 64) as u64 as i64);
-                            fx.bcx.ins().iconcat(lsb, msb)
-                        }
-                        ty => fx.bcx.ins().iconst(ty, delta as i64),
-                    };
-                    fx.bcx.ins().iadd(tag, delta)
-                };
-
-                let cast_tag = clif_intcast(fx, incr_tag, cast_to, !niches_ule);
-
-                (is_niche, cast_tag, 0)
             } else {
                 // The special cases don't apply, so we'll have to go with
                 // the general algorithm.
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index 7c6fd9f6f..3e2e2af96 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -377,7 +377,7 @@ pub(crate) fn run_aot(
     };
 
     if tcx.dep_graph.is_fully_enabled() {
-        for cgu in &*cgus {
+        for cgu in cgus {
             tcx.ensure().codegen_unit(cgu.name());
         }
     }
@@ -417,7 +417,7 @@ pub(crate) fn run_aot(
                     CguReuse::PreLto => unreachable!(),
                     CguReuse::PostLto => {
                         concurrency_limiter.job_already_done();
-                        OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, &*cgu))
+                        OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu))
                     }
                 }
             })
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index 8b5a2da2c..f6a48e325 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -311,7 +311,11 @@ fn dep_symbol_lookup_fn(
         .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable)
         .unwrap()
         .1;
-    for &cnum in &crate_info.used_crates {
+    // `used_crates` is in reverse postorder in terms of dependencies. Reverse the order here to
+    // get a postorder which ensures that all dependencies of a dylib are loaded before the dylib
+    // itself. This helps the dynamic linker to find dylibs not in the regular dynamic library
+    // search path.
+    for &cnum in crate_info.used_crates.iter().rev() {
         let src = &crate_info.used_crate_source[&cnum];
         match data[cnum.as_usize() - 1] {
             Linkage::NotLinked | Linkage::IncludedFromDylib => {}
diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs
index 46c78ce6a..a74f8ffa2 100644
--- a/compiler/rustc_codegen_cranelift/src/global_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs
@@ -125,7 +125,7 @@ pub(crate) fn compile_global_asm(
     let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name));
 
     // Assemble `global_asm`
-    let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
+    let global_asm_object_file = add_file_stem_postfix(output_object_file, ".asm");
     let mut child = Command::new(&config.assembler)
         .arg("-o")
         .arg(&global_asm_object_file)
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index 6206fbf7d..3ba530c04 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -242,7 +242,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                 }
             }
             InlineAsmOperand::Const { ref value } => {
-                let (const_value, ty) = crate::constant::eval_mir_constant(fx, &*value)
+                let (const_value, ty) = crate::constant::eval_mir_constant(fx, value)
                     .unwrap_or_else(|| span_bug!(span, "asm const cannot be resolved"));
                 let value = rustc_codegen_ssa::common::asm_const_to_str(
                     fx.tcx,
@@ -334,13 +334,13 @@ pub(crate) fn codegen_inline_asm<'tcx>(
             }
             CInlineAsmOperand::Out { reg: _, late: _, place } => {
                 if let Some(place) = place {
-                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), place.clone()));
+                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place));
                 }
             }
             CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => {
                 inputs.push((asm_gen.stack_slots_input[i].unwrap(), in_value.load_scalar(fx)));
                 if let Some(out_place) = out_place {
-                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), out_place.clone()));
+                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), *out_place));
                 }
             }
             CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {}
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
index e5c4b244a..0f32d1a25 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
@@ -179,8 +179,8 @@ fn llvm_add_sub<'tcx>(
 
     // c + carry -> c + first intermediate carry or borrow respectively
     let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b);
-    let c = int0.value_field(fx, mir::Field::new(0));
-    let cb0 = int0.value_field(fx, mir::Field::new(1)).load_scalar(fx);
+    let c = int0.value_field(fx, FieldIdx::new(0));
+    let cb0 = int0.value_field(fx, FieldIdx::new(1)).load_scalar(fx);
 
     // c + carry -> c + second intermediate carry or borrow respectively
     let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in);
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index e74aabf2f..03f2a65fc 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -23,7 +23,7 @@ pub(crate) use llvm::codegen_llvm_intrinsic_call;
 
 use rustc_middle::ty;
 use rustc_middle::ty::layout::{HasParamEnv, ValidityRequirement};
-use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_span::symbol::{kw, sym, Symbol};
 
@@ -252,41 +252,45 @@ fn codegen_float_intrinsic_call<'tcx>(
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
 ) -> bool {
-    let (name, arg_count, ty) = match intrinsic {
-        sym::expf32 => ("expf", 1, fx.tcx.types.f32),
-        sym::expf64 => ("exp", 1, fx.tcx.types.f64),
-        sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32),
-        sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64),
-        sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32),
-        sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64),
-        sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32), // compiler-builtins
-        sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64), // compiler-builtins
-        sym::powf32 => ("powf", 2, fx.tcx.types.f32),
-        sym::powf64 => ("pow", 2, fx.tcx.types.f64),
-        sym::logf32 => ("logf", 1, fx.tcx.types.f32),
-        sym::logf64 => ("log", 1, fx.tcx.types.f64),
-        sym::log2f32 => ("log2f", 1, fx.tcx.types.f32),
-        sym::log2f64 => ("log2", 1, fx.tcx.types.f64),
-        sym::log10f32 => ("log10f", 1, fx.tcx.types.f32),
-        sym::log10f64 => ("log10", 1, fx.tcx.types.f64),
-        sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32),
-        sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64),
-        sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32),
-        sym::fmaf64 => ("fma", 3, fx.tcx.types.f64),
-        sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32),
-        sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64),
-        sym::floorf32 => ("floorf", 1, fx.tcx.types.f32),
-        sym::floorf64 => ("floor", 1, fx.tcx.types.f64),
-        sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32),
-        sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64),
-        sym::truncf32 => ("truncf", 1, fx.tcx.types.f32),
-        sym::truncf64 => ("trunc", 1, fx.tcx.types.f64),
-        sym::roundf32 => ("roundf", 1, fx.tcx.types.f32),
-        sym::roundf64 => ("round", 1, fx.tcx.types.f64),
-        sym::sinf32 => ("sinf", 1, fx.tcx.types.f32),
-        sym::sinf64 => ("sin", 1, fx.tcx.types.f64),
-        sym::cosf32 => ("cosf", 1, fx.tcx.types.f32),
-        sym::cosf64 => ("cos", 1, fx.tcx.types.f64),
+    let (name, arg_count, ty, clif_ty) = match intrinsic {
+        sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32),
+        sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64),
+        sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32),
+        sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64),
+        sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32),
+        sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64),
+        sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins
+        sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins
+        sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32),
+        sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64),
+        sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32),
+        sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64),
+        sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32),
+        sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64),
+        sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32),
+        sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64),
+        sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32, types::F32),
+        sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64, types::F64),
+        sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
+        sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
+        sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32),
+        sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64),
+        sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32),
+        sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64),
+        sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32),
+        sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64),
+        sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32),
+        sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64),
+        sym::rintf32 => ("rintf", 1, fx.tcx.types.f32, types::F32),
+        sym::rintf64 => ("rint", 1, fx.tcx.types.f64, types::F64),
+        sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32),
+        sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64),
+        sym::roundevenf32 => ("roundevenf", 1, fx.tcx.types.f32, types::F32),
+        sym::roundevenf64 => ("roundeven", 1, fx.tcx.types.f64, types::F64),
+        sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32),
+        sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64),
+        sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32),
+        sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64),
         _ => return false,
     };
 
@@ -297,15 +301,19 @@ fn codegen_float_intrinsic_call<'tcx>(
     let (a, b, c);
     let args = match args {
         [x] => {
-            a = [codegen_operand(fx, x)];
+            a = [codegen_operand(fx, x).load_scalar(fx)];
             &a as &[_]
         }
         [x, y] => {
-            b = [codegen_operand(fx, x), codegen_operand(fx, y)];
+            b = [codegen_operand(fx, x).load_scalar(fx), codegen_operand(fx, y).load_scalar(fx)];
             &b
         }
         [x, y, z] => {
-            c = [codegen_operand(fx, x), codegen_operand(fx, y), codegen_operand(fx, z)];
+            c = [
+                codegen_operand(fx, x).load_scalar(fx),
+                codegen_operand(fx, y).load_scalar(fx),
+                codegen_operand(fx, z).load_scalar(fx),
+            ];
             &c
         }
         _ => unreachable!(),
@@ -314,15 +322,10 @@ fn codegen_float_intrinsic_call<'tcx>(
     let layout = fx.layout_of(ty);
     let res = match intrinsic {
         sym::fmaf32 | sym::fmaf64 => {
-            let a = args[0].load_scalar(fx);
-            let b = args[1].load_scalar(fx);
-            let c = args[2].load_scalar(fx);
-            CValue::by_val(fx.bcx.ins().fma(a, b, c), layout)
+            CValue::by_val(fx.bcx.ins().fma(args[0], args[1], args[2]), layout)
         }
         sym::copysignf32 | sym::copysignf64 => {
-            let a = args[0].load_scalar(fx);
-            let b = args[1].load_scalar(fx);
-            CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout)
+            CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout)
         }
         sym::fabsf32
         | sym::fabsf64
@@ -332,21 +335,29 @@ fn codegen_float_intrinsic_call<'tcx>(
         | sym::ceilf64
         | sym::truncf32
         | sym::truncf64 => {
-            let a = args[0].load_scalar(fx);
-
             let val = match intrinsic {
-                sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a),
-                sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a),
-                sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a),
-                sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a),
+                sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(args[0]),
+                sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
+                sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
+                sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
                 _ => unreachable!(),
             };
 
             CValue::by_val(val, layout)
         }
+
         // These intrinsics aren't supported natively by Cranelift.
         // Lower them to a libcall.
-        _ => fx.easy_call(name, &args, ty),
+        sym::powif32 | sym::powif64 => {
+            let input_tys: Vec<_> = vec![AbiParam::new(clif_ty), AbiParam::new(types::I32)];
+            let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
+            CValue::by_val(ret_val, fx.layout_of(ty))
+        }
+        _ => {
+            let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect();
+            let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
+            CValue::by_val(ret_val, fx.layout_of(ty))
+        }
     };
 
     ret.write_cvalue(fx, res);
@@ -381,7 +392,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
             fx.bcx.ins().debugtrap();
         }
-        sym::copy | sym::copy_nonoverlapping => {
+        sym::copy => {
             intrinsic_args!(fx, args => (src, dst, count); intrinsic);
             let src = src.load_scalar(fx);
             let dst = dst.load_scalar(fx);
@@ -393,13 +404,8 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let byte_amount =
                 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
 
-            if intrinsic == sym::copy_nonoverlapping {
-                // FIXME emit_small_memcpy
-                fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
-            } else {
-                // FIXME emit_small_memmove
-                fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
-            }
+            // FIXME emit_small_memmove
+            fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
         }
         sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => {
             // NOTE: the volatile variants have src and dst swapped
@@ -551,16 +557,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
             fx.bcx.ins().band(ptr, mask);
         }
 
-        sym::transmute => {
-            intrinsic_args!(fx, args => (from); intrinsic);
-
-            if ret.layout().abi.is_uninhabited() {
-                crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
-                return;
-            }
-
-            ret.write_cvalue_transmute(fx, from);
-        }
         sym::write_bytes | sym::volatile_set_memory => {
             intrinsic_args!(fx, args => (dst, val, count); intrinsic);
             let val = val.load_scalar(fx);
@@ -639,26 +635,25 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
                 if do_panic {
                     let layout = fx.layout_of(ty);
-
-                    with_no_trimmed_paths!({
-                        crate::base::codegen_panic_nounwind(
-                            fx,
-                            &if layout.abi.is_uninhabited() {
-                                format!("attempted to instantiate uninhabited type `{}`", layout.ty)
-                            } else if requirement == ValidityRequirement::Zero {
+                    let msg_str = with_no_visible_paths!({
+                        with_no_trimmed_paths!({
+                            if layout.abi.is_uninhabited() {
+                                // Use this error even for the other intrinsics as it is more precise.
+                                format!("attempted to instantiate uninhabited type `{}`", ty)
+                            } else if intrinsic == sym::assert_zero_valid {
                                 format!(
                                     "attempted to zero-initialize type `{}`, which is invalid",
-                                    layout.ty
+                                    ty
                                 )
                             } else {
                                 format!(
                                     "attempted to leave type `{}` uninitialized, which is invalid",
-                                    layout.ty
+                                    ty
                                 )
-                            },
-                            source_info,
-                        )
+                            }
+                        })
                     });
+                    crate::base::codegen_panic_nounwind(fx, &msg_str, source_info);
                     return;
                 }
             }
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index a1d63acfb..6f54a8d49 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -253,7 +253,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
 
             ret.write_cvalue(fx, base);
-            let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
+            let ret_lane = ret.place_field(fx, FieldIdx::new(idx.try_into().unwrap()));
             ret_lane.write_cvalue(fx, val);
         }
 
@@ -279,9 +279,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
                 let trap_block = fx.bcx.create_block();
                 let true_ = fx.bcx.ins().iconst(types::I8, 1);
-                fx.bcx.ins().brnz(true_, trap_block, &[]);
                 let ret_block = fx.get_block(target);
-                fx.bcx.ins().jump(ret_block, &[]);
+                fx.bcx.ins().brif(true_, trap_block, &[], ret_block, &[]);
                 fx.bcx.switch_to_block(trap_block);
                 crate::trap::trap_unimplemented(
                     fx,
@@ -825,8 +824,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let next = fx.bcx.create_block();
                 let res_lane = fx.bcx.append_block_param(next, lane_clif_ty);
 
-                fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
-                fx.bcx.ins().jump(if_disabled, &[]);
+                fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]);
                 fx.bcx.seal_block(if_enabled);
                 fx.bcx.seal_block(if_disabled);
 
@@ -864,8 +862,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let if_enabled = fx.bcx.create_block();
                 let next = fx.bcx.create_block();
 
-                fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
-                fx.bcx.ins().jump(next, &[]);
+                fx.bcx.ins().brif(mask_lane, if_enabled, &[], next, &[]);
                 fx.bcx.seal_block(if_enabled);
 
                 fx.bcx.switch_to_block(if_enabled);
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index 80ce3dc93..8cc7f6c34 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -57,8 +57,6 @@ mod compiler_builtins;
 mod concurrency_limiter;
 mod config;
 mod constant;
-// FIXME revert back to the external crate with Cranelift 0.93
-mod cranelift_native;
 mod debuginfo;
 mod discriminant;
 mod driver;
@@ -88,7 +86,7 @@ mod prelude {
         self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
         TypeFoldable, TypeVisitableExt, UintTy,
     };
-    pub(crate) use rustc_target::abi::{Abi, Scalar, Size, VariantIdx};
+    pub(crate) use rustc_target::abi::{Abi, FieldIdx, Scalar, Size, VariantIdx, FIRST_VARIANT};
 
     pub(crate) use rustc_data_structures::fx::FxHashMap;
 
@@ -251,7 +249,7 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple {
     }
 }
 
-fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::TargetIsa + 'static> {
+fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::TargetIsa + 'static> {
     use target_lexicon::BinaryFormat;
 
     let target_triple = crate::target_triple(sess);
@@ -285,14 +283,17 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
         }
     }
 
-    if let target_lexicon::Architecture::Aarch64(_) | target_lexicon::Architecture::X86_64 =
-        target_triple.architecture
+    if let target_lexicon::Architecture::Aarch64(_)
+    | target_lexicon::Architecture::Riscv64(_)
+    | target_lexicon::Architecture::X86_64 = target_triple.architecture
     {
-        // Windows depends on stack probes to grow the committed part of the stack
+        // Windows depends on stack probes to grow the committed part of the stack.
+        // On other platforms it helps prevents stack smashing.
         flags_builder.enable("enable_probestack").unwrap();
         flags_builder.set("probestack_strategy", "inline").unwrap();
     } else {
-        // __cranelift_probestack is not provided and inline stack probes are only supported on AArch64 and x86_64
+        // __cranelift_probestack is not provided and inline stack probes are only supported on
+        // AArch64, Riscv64 and x86_64.
         flags_builder.set("enable_probestack", "false").unwrap();
     }
 
diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs
index be908df83..205411e8c 100644
--- a/compiler/rustc_codegen_cranelift/src/main_shim.rs
+++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs
@@ -28,7 +28,7 @@ pub(crate) fn maybe_create_entry_wrapper(
 
     if main_def_id.is_local() {
         let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx);
-        if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() {
+        if !is_jit && module.get_name(tcx.symbol_name(instance).name).is_none() {
             return;
         }
     } else if !is_primary_cgu {
diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs
index c058ece96..1357b7be1 100644
--- a/compiler/rustc_codegen_cranelift/src/num.rs
+++ b/compiler/rustc_codegen_cranelift/src/num.rs
@@ -170,14 +170,6 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
 ) -> CValue<'tcx> {
-    if bin_op != BinOp::Shl && bin_op != BinOp::Shr {
-        assert_eq!(
-            in_lhs.layout().ty,
-            in_rhs.layout().ty,
-            "checked int binop requires lhs and rhs of same type"
-        );
-    }
-
     let lhs = in_lhs.load_scalar(fx);
     let rhs = in_rhs.load_scalar(fx);
 
@@ -271,21 +263,6 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
                 _ => unreachable!("invalid non-integer type {}", ty),
             }
         }
-        BinOp::Shl => {
-            let val = fx.bcx.ins().ishl(lhs, rhs);
-            let ty = fx.bcx.func.dfg.value_type(val);
-            let max_shift = i64::from(ty.bits()) - 1;
-            let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
-            (val, has_overflow)
-        }
-        BinOp::Shr => {
-            let val =
-                if !signed { fx.bcx.ins().ushr(lhs, rhs) } else { fx.bcx.ins().sshr(lhs, rhs) };
-            let ty = fx.bcx.func.dfg.value_type(val);
-            let max_shift = i64::from(ty.bits()) - 1;
-            let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
-            (val, has_overflow)
-        }
         _ => bug!("binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", bin_op, in_lhs, in_rhs),
     };
 
@@ -347,12 +324,20 @@ pub(crate) fn codegen_float_binop<'tcx>(
         BinOp::Mul => b.fmul(lhs, rhs),
         BinOp::Div => b.fdiv(lhs, rhs),
         BinOp::Rem => {
-            let name = match in_lhs.layout().ty.kind() {
-                ty::Float(FloatTy::F32) => "fmodf",
-                ty::Float(FloatTy::F64) => "fmod",
+            let (name, ty) = match in_lhs.layout().ty.kind() {
+                ty::Float(FloatTy::F32) => ("fmodf", types::F32),
+                ty::Float(FloatTy::F64) => ("fmod", types::F64),
                 _ => bug!(),
             };
-            return fx.easy_call(name, &[in_lhs, in_rhs], in_lhs.layout().ty);
+
+            let ret_val = fx.lib_call(
+                name,
+                vec![AbiParam::new(ty), AbiParam::new(ty)],
+                vec![AbiParam::new(ty)],
+                &[lhs, rhs],
+            )[0];
+
+            return CValue::by_val(ret_val, in_lhs.layout());
         }
         BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
             let fltcc = match bin_op {
diff --git a/compiler/rustc_codegen_cranelift/src/pointer.rs b/compiler/rustc_codegen_cranelift/src/pointer.rs
index 31d827f83..b60e56720 100644
--- a/compiler/rustc_codegen_cranelift/src/pointer.rs
+++ b/compiler/rustc_codegen_cranelift/src/pointer.rs
@@ -30,11 +30,6 @@ impl Pointer {
         Pointer { base: PointerBase::Stack(stack_slot), offset: Offset32::new(0) }
     }
 
-    pub(crate) fn const_addr(fx: &mut FunctionCx<'_, '_, '_>, addr: i64) -> Self {
-        let addr = fx.bcx.ins().iconst(fx.pointer_type, addr);
-        Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
-    }
-
     pub(crate) fn dangling(align: Align) -> Self {
         Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
     }
diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
index a7af16268..e0a081c9d 100644
--- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
@@ -245,7 +245,7 @@ pub(crate) fn write_clif_file(
         for flag in isa.flags().iter() {
             writeln!(file, "set {}", flag)?;
         }
-        write!(file, "target {}", isa.triple().architecture.to_string())?;
+        write!(file, "target {}", isa.triple().architecture)?;
         for isa_flag in isa.isa_flags().iter() {
             write!(file, " {}", isa_flag)?;
         }
diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs
index a0745582d..ff0e12410 100644
--- a/compiler/rustc_codegen_cranelift/src/unsize.rs
+++ b/compiler/rustc_codegen_cranelift/src/unsize.rs
@@ -28,9 +28,7 @@ pub(crate) fn unsized_info<'tcx>(
         (
             &ty::Dynamic(ref data_a, _, src_dyn_kind),
             &ty::Dynamic(ref data_b, _, target_dyn_kind),
-        ) => {
-            assert_eq!(src_dyn_kind, target_dyn_kind);
-
+        ) if src_dyn_kind == target_dyn_kind => {
             let old_info =
                 old_info.expect("unsized_info: missing old info for trait upcasting coercion");
             if data_a.principal_def_id() == data_b.principal_def_id() {
@@ -55,7 +53,7 @@ pub(crate) fn unsized_info<'tcx>(
                 old_info
             }
         }
-        (_, &ty::Dynamic(ref data, ..)) => crate::vtable::get_vtable(fx, source, data.principal()),
+        (_, ty::Dynamic(data, ..)) => crate::vtable::get_vtable(fx, source, data.principal()),
         _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target),
     }
 }
@@ -148,9 +146,9 @@ pub(crate) fn coerce_unsized_into<'tcx>(
         (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {
             assert_eq!(def_a, def_b);
 
-            for i in 0..def_a.variant(VariantIdx::new(0)).fields.len() {
-                let src_f = src.value_field(fx, mir::Field::new(i));
-                let dst_f = dst.place_field(fx, mir::Field::new(i));
+            for i in 0..def_a.variant(FIRST_VARIANT).fields.len() {
+                let src_f = src.value_field(fx, FieldIdx::new(i));
+                let dst_f = dst.place_field(fx, FieldIdx::new(i));
 
                 if dst_f.layout().is_zst() {
                     continue;
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index cc1edaa97..1b69862ce 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -3,13 +3,14 @@
 use crate::prelude::*;
 
 use cranelift_codegen::ir::immediates::Offset32;
+use cranelift_codegen::ir::{InstructionData, Opcode};
 
 fn codegen_field<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     base: Pointer,
     extra: Option<Value>,
     layout: TyAndLayout<'tcx>,
-    field: mir::Field,
+    field: FieldIdx,
 ) -> (Pointer, TyAndLayout<'tcx>) {
     let field_offset = layout.fields.offset(field.index());
     let field_layout = layout.field(&*fx, field.index());
@@ -209,7 +210,7 @@ impl<'tcx> CValue<'tcx> {
     pub(crate) fn value_field(
         self,
         fx: &mut FunctionCx<'_, '_, 'tcx>,
-        field: mir::Field,
+        field: FieldIdx,
     ) -> CValue<'tcx> {
         let layout = self.1;
         match self.0 {
@@ -457,6 +458,7 @@ impl<'tcx> CPlace<'tcx> {
         }
     }
 
+    #[track_caller]
     pub(crate) fn to_ptr(self) -> Pointer {
         match self.to_ptr_maybe_unsized() {
             (ptr, None) => ptr,
@@ -464,6 +466,7 @@ impl<'tcx> CPlace<'tcx> {
         }
     }
 
+    #[track_caller]
     pub(crate) fn to_ptr_maybe_unsized(self) -> (Pointer, Option<Value>) {
         match self.inner {
             CPlaceInner::Addr(ptr, extra) => (ptr, extra),
@@ -684,7 +687,7 @@ impl<'tcx> CPlace<'tcx> {
     pub(crate) fn place_field(
         self,
         fx: &mut FunctionCx<'_, '_, 'tcx>,
-        field: mir::Field,
+        field: FieldIdx,
     ) -> CPlace<'tcx> {
         let layout = self.layout();
 
@@ -698,7 +701,8 @@ impl<'tcx> CPlace<'tcx> {
                     };
                 }
                 ty::Adt(adt_def, substs) if layout.ty.is_simd() => {
-                    let f0_ty = adt_def.non_enum_variant().fields[0].ty(fx.tcx, substs);
+                    let f0 = &adt_def.non_enum_variant().fields[FieldIdx::from_u32(0)];
+                    let f0_ty = f0.ty(fx.tcx, substs);
 
                     match f0_ty.kind() {
                         ty::Array(_, _) => {
@@ -787,7 +791,36 @@ impl<'tcx> CPlace<'tcx> {
         index: Value,
     ) -> CPlace<'tcx> {
         let (elem_layout, ptr) = match self.layout().ty.kind() {
-            ty::Array(elem_ty, _) => (fx.layout_of(*elem_ty), self.to_ptr()),
+            ty::Array(elem_ty, _) => {
+                let elem_layout = fx.layout_of(*elem_ty);
+                match self.inner {
+                    CPlaceInner::Var(local, var) => {
+                        // This is a hack to handle `vector_val.0[1]`. It doesn't allow dynamic
+                        // indexing.
+                        let lane_idx = match fx.bcx.func.dfg.insts
+                            [fx.bcx.func.dfg.value_def(index).unwrap_inst()]
+                        {
+                            InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => imm,
+                            _ => bug!(
+                                "Dynamic indexing into a vector type is not supported: {self:?}[{index}]"
+                            ),
+                        };
+                        return CPlace {
+                            inner: CPlaceInner::VarLane(
+                                local,
+                                var,
+                                lane_idx.bits().try_into().unwrap(),
+                            ),
+                            layout: elem_layout,
+                        };
+                    }
+                    CPlaceInner::Addr(addr, None) => (elem_layout, addr),
+                    CPlaceInner::Addr(_, Some(_))
+                    | CPlaceInner::VarPair(_, _, _)
+                    | CPlaceInner::VarLane(_, _, _) => bug!("Can't index into {self:?}"),
+                }
+                // FIXME use VarLane in case of Var with simd type
+            }
             ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_maybe_unsized().0),
             _ => bug!("place_index({:?})", self.layout().ty),
         };
diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs
index f04fb82de..b309695c1 100644
--- a/compiler/rustc_codegen_cranelift/src/vtable.rs
+++ b/compiler/rustc_codegen_cranelift/src/vtable.rs
@@ -43,17 +43,34 @@ pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -
 
 pub(crate) fn get_ptr_and_method_ref<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    arg: CValue<'tcx>,
+    mut arg: CValue<'tcx>,
     idx: usize,
 ) -> (Pointer, Value) {
     let (ptr, vtable) = 'block: {
+        if let Abi::Scalar(_) = arg.layout().abi {
+            'descend_newtypes: while !arg.layout().ty.is_unsafe_ptr() && !arg.layout().ty.is_ref() {
+                for i in 0..arg.layout().fields.count() {
+                    let field = arg.value_field(fx, FieldIdx::new(i));
+                    if !field.layout().is_zst() {
+                        // we found the one non-zero-sized field that is allowed
+                        // now find *its* non-zero-sized field, or stop if it's a
+                        // pointer
+                        arg = field;
+                        continue 'descend_newtypes;
+                    }
+                }
+
+                bug!("receiver has no non-zero-sized fields {:?}", arg);
+            }
+        }
+
         if let ty::Ref(_, ty, _) = arg.layout().ty.kind() {
             if ty.is_dyn_star() {
                 let inner_layout = fx.layout_of(arg.layout().ty.builtin_deref(true).unwrap().ty);
                 let dyn_star = CPlace::for_ptr(Pointer::new(arg.load_scalar(fx)), inner_layout);
-                let ptr = dyn_star.place_field(fx, mir::Field::new(0)).to_ptr();
+                let ptr = dyn_star.place_field(fx, FieldIdx::new(0)).to_ptr();
                 let vtable =
-                    dyn_star.place_field(fx, mir::Field::new(1)).to_cvalue(fx).load_scalar(fx);
+                    dyn_star.place_field(fx, FieldIdx::new(1)).to_cvalue(fx).load_scalar(fx);
                 break 'block (ptr, vtable);
             }
         }
diff --git a/compiler/rustc_codegen_gcc/.github/workflows/ci.yml b/compiler/rustc_codegen_gcc/.github/workflows/ci.yml
index 8ebdabe82..d2b7724a2 100644
--- a/compiler/rustc_codegen_gcc/.github/workflows/ci.yml
+++ b/compiler/rustc_codegen_gcc/.github/workflows/ci.yml
@@ -4,36 +4,72 @@ on:
   - push
   - pull_request
 
+permissions:
+  contents: read
+
+env:
+  # Enable backtraces for easier debugging
+  RUST_BACKTRACE: 1
+
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
       matrix:
-        libgccjit_version: ["libgccjit.so", "libgccjit_without_int128.so", "libgccjit12.so"]
+        libgccjit_version:
+          - { gcc: "libgccjit.so", extra: "", env_extra: "", artifacts_branch: "master" }
+          - { gcc: "libgccjit_without_int128.so", extra: "", env_extra: "", artifacts_branch: "master-without-128bit-integers" }
+          - { gcc: "libgccjit12.so", extra: "--no-default-features", env_extra: "TEST_FLAGS='-Cpanic=abort -Zpanic-abort-tests'", artifacts_branch: "gcc12" }
+        commands: [
+          "--mini-tests",
+          "--std-tests",
+          # FIXME: re-enable asm tests when GCC can emit in the right syntax.
+          # "--asm-tests",
+          "--test-libcore",
+          "--extended-rand-tests",
+          "--extended-regex-example-tests",
+          "--extended-regex-tests",
+          "--test-successful-rustc --nb-parts 2 --current-part 0",
+          "--test-successful-rustc --nb-parts 2 --current-part 1",
+          "--test-failing-rustc",
+        ]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         repository: llvm/llvm-project
         path: llvm
 
     - name: Install packages
-      run: sudo apt-get install ninja-build ripgrep
+      # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
+      run: sudo apt-get install ninja-build ripgrep llvm-14-tools
+
+    - name: Install libgccjit12
+      if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
+      run: sudo apt-get install libgccjit-12-dev
 
     - name: Download artifact
+      if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
       uses: dawidd6/action-download-artifact@v2
       with:
           workflow: main.yml
-          name: ${{ matrix.libgccjit_version }}
+          name: ${{ matrix.libgccjit_version.gcc }}
           path: gcc-build
           repo: antoyo/gcc
+          branch: ${{ matrix.libgccjit_version.artifacts_branch }}
+          event: push
           search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
 
     - name: Setup path to libgccjit
+      if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
+      run: echo /usr/lib/gcc/x86_64-linux-gnu/12 > gcc_path
+
+    - name: Setup path to libgccjit
+      if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
       run: |
           echo $(readlink -f gcc-build) > gcc_path
           # NOTE: the filename is still libgccjit.so even when the artifact name is different.
@@ -48,49 +84,44 @@ jobs:
     - name: Set RUST_COMPILER_RT_ROOT
       run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
 
-    # https://github.com/actions/cache/issues/133
-    - name: Fixup owner of ~/.cargo/
-      # Don't remove the trailing /. It is necessary to follow the symlink.
-      run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/
-
     - name: Cache cargo installed crates
-      uses: actions/cache@v1.1.2
+      uses: actions/cache@v3
       with:
         path: ~/.cargo/bin
         key: cargo-installed-crates2-ubuntu-latest
 
     - name: Cache cargo registry
-      uses: actions/cache@v1
+      uses: actions/cache@v3
       with:
         path: ~/.cargo/registry
         key: ${{ runner.os }}-cargo-registry2-${{ hashFiles('**/Cargo.lock') }}
 
     - name: Cache cargo index
-      uses: actions/cache@v1
+      uses: actions/cache@v3
       with:
         path: ~/.cargo/git
         key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
 
     - name: Cache cargo target dir
-      uses: actions/cache@v1.1.2
+      uses: actions/cache@v3
       with:
         path: target
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain') }}
 
-    - name: Build
-      if: matrix.libgccjit_version != 'libgccjit12.so'
-      run: |
-        ./prepare_build.sh
-        ./build.sh
-        cargo test
-        ./clean_all.sh
+    #- name: Cache rust repository
+      ## We only clone the rust repository for rustc tests
+      #if: ${{ contains(matrix.commands, 'rustc') }}
+      #uses: actions/cache@v3
+      #id: cache-rust-repository
+      #with:
+        #path: rust
+        #key: ${{ runner.os }}-packages-${{ hashFiles('rust/.git/HEAD') }}
 
     - name: Build
-      if: matrix.libgccjit_version == 'libgccjit12.so'
       run: |
         ./prepare_build.sh
-        ./build.sh --no-default-features
-        cargo test --no-default-features
+        ${{ matrix.libgccjit_version.env_extra }} ./build.sh ${{ matrix.libgccjit_version.extra }}
+        ${{ matrix.libgccjit_version.env_extra }} cargo test ${{ matrix.libgccjit_version.extra }}
         ./clean_all.sh
 
     - name: Prepare dependencies
@@ -106,26 +137,16 @@ jobs:
         command: build
         args: --release
 
-    - name: Test
-      if: matrix.libgccjit_version != 'libgccjit12.so'
-      run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
-
-        # Reduce amount of benchmark runs as they are slow
-        export COMPILE_RUNS=2
-        export RUN_RUNS=2
+    - name: Add more failing tests for GCC 12
+      if: ${{ matrix.libgccjit_version.gcc == 'libgccjit12.so' }}
+      run: cat failing-ui-tests12.txt >> failing-ui-tests.txt
 
-        ./test.sh --release
-
-    - name: Test
-      if: matrix.libgccjit_version == 'libgccjit12.so'
+    - name: Run tests
       run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
-
-        # Reduce amount of benchmark runs as they are slow
-        export COMPILE_RUNS=2
-        export RUN_RUNS=2
+        ${{ matrix.libgccjit_version.env_extra }} ./test.sh --release --clean --build-sysroot ${{ matrix.commands }} ${{ matrix.libgccjit_version.extra }}
 
-        ./test.sh --release --no-default-features
+  duplicates:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: python tools/check_intrinsics_duplicates.py
diff --git a/compiler/rustc_codegen_gcc/.github/workflows/release.yml b/compiler/rustc_codegen_gcc/.github/workflows/release.yml
new file mode 100644
index 000000000..c4e99469b
--- /dev/null
+++ b/compiler/rustc_codegen_gcc/.github/workflows/release.yml
@@ -0,0 +1,111 @@
+name: CI with sysroot compiled in release mode
+
+on:
+  - push
+  - pull_request
+
+permissions:
+  contents: read
+
+env:
+  # Enable backtraces for easier debugging
+  RUST_BACKTRACE: 1
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        libgccjit_version:
+          - { gcc: "libgccjit.so", artifacts_branch: "master" }
+        commands: [
+          "--test-successful-rustc --nb-parts 2 --current-part 0",
+          "--test-successful-rustc --nb-parts 2 --current-part 1",
+        ]
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/checkout@v3
+      with:
+        repository: llvm/llvm-project
+        path: llvm
+
+    - name: Install packages
+      run: sudo apt-get install ninja-build ripgrep
+
+    - name: Download artifact
+      uses: dawidd6/action-download-artifact@v2
+      with:
+          workflow: main.yml
+          name: ${{ matrix.libgccjit_version.gcc }}
+          path: gcc-build
+          repo: antoyo/gcc
+          branch: ${{ matrix.libgccjit_version.artifacts_branch }}
+          event: push
+          search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
+
+    - name: Setup path to libgccjit
+      run: |
+          echo $(readlink -f gcc-build) > gcc_path
+          # NOTE: the filename is still libgccjit.so even when the artifact name is different.
+          ln gcc-build/libgccjit.so gcc-build/libgccjit.so.0
+
+    - name: Set env
+      run: |
+        echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
+
+    - name: Set RUST_COMPILER_RT_ROOT
+      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin
+        key: cargo-installed-crates2-ubuntu-latest
+
+    - name: Cache cargo registry
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/registry
+        key: ${{ runner.os }}-cargo-registry2-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/git
+        key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain') }}
+
+    - name: Build
+      run: |
+        ./prepare_build.sh
+        ./build.sh --release --release-sysroot
+        cargo test
+        ./clean_all.sh
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    # Compile is a separate step, as the actions-rs/cargo action supports error annotations
+    - name: Compile
+      uses: actions-rs/cargo@v1.0.3
+      with:
+        command: build
+        args: --release
+
+    - name: Run tests
+      run: |
+        ./test.sh --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }}
diff --git a/compiler/rustc_codegen_gcc/.github/workflows/stdarch.yml b/compiler/rustc_codegen_gcc/.github/workflows/stdarch.yml
new file mode 100644
index 000000000..42fb35e73
--- /dev/null
+++ b/compiler/rustc_codegen_gcc/.github/workflows/stdarch.yml
@@ -0,0 +1,116 @@
+name: stdarch tests with sysroot compiled in release mode
+
+on:
+  - push
+  - pull_request
+
+permissions:
+  contents: read
+
+env:
+  # Enable backtraces for easier debugging
+  RUST_BACKTRACE: 1
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        libgccjit_version:
+          - { gcc: "libgccjit.so", artifacts_branch: "master" }
+        commands: [
+          "--test-successful-rustc --nb-parts 2 --current-part 0",
+          "--test-successful-rustc --nb-parts 2 --current-part 1",
+        ]
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/checkout@v3
+      with:
+        repository: llvm/llvm-project
+        path: llvm
+
+    - name: Install packages
+      run: sudo apt-get install ninja-build ripgrep
+
+    - name: Download artifact
+      uses: dawidd6/action-download-artifact@v2
+      with:
+          workflow: main.yml
+          name: ${{ matrix.libgccjit_version.gcc }}
+          path: gcc-build
+          repo: antoyo/gcc
+          branch: ${{ matrix.libgccjit_version.artifacts_branch }}
+          event: push
+          search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
+
+    - name: Setup path to libgccjit
+      run: |
+          echo $(readlink -f gcc-build) > gcc_path
+          # NOTE: the filename is still libgccjit.so even when the artifact name is different.
+          ln gcc-build/libgccjit.so gcc-build/libgccjit.so.0
+
+    - name: Set env
+      run: |
+        echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
+
+    - name: Set RUST_COMPILER_RT_ROOT
+      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin
+        key: cargo-installed-crates2-ubuntu-latest
+
+    - name: Cache cargo registry
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/registry
+        key: ${{ runner.os }}-cargo-registry2-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/git
+        key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain') }}
+
+    - name: Build
+      run: |
+        ./prepare_build.sh
+        ./build.sh --release --release-sysroot
+        cargo test
+        ./clean_all.sh
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    # Compile is a separate step, as the actions-rs/cargo action supports error annotations
+    - name: Compile
+      uses: actions-rs/cargo@v1.0.3
+      with:
+        command: build
+        args: --release
+
+    - name: Run tests
+      run: |
+        ./test.sh --release --clean --release-sysroot --build-sysroot --mini-tests --std-tests --test-libcore
+
+    - name: Run stdarch tests
+      run: |
+        cd build_sysroot/sysroot_src/library/stdarch/
+        CHANNEL=release TARGET=x86_64-unknown-linux-gnu ../../../../cargo.sh test
diff --git a/compiler/rustc_codegen_gcc/Cargo.lock b/compiler/rustc_codegen_gcc/Cargo.lock
index 1cb219e12..0f2e152f8 100644
--- a/compiler/rustc_codegen_gcc/Cargo.lock
+++ b/compiler/rustc_codegen_gcc/Cargo.lock
@@ -35,7 +35,7 @@ dependencies = [
 [[package]]
 name = "gccjit"
 version = "1.0.0"
-source = "git+https://github.com/antoyo/gccjit.rs#bdb86fb5092895ff5589726b33250010c64d93f6"
+source = "git+https://github.com/antoyo/gccjit.rs#fe242b7eb26980e6c78859d51c8d4cc1e43381a3"
 dependencies = [
  "gccjit_sys",
 ]
@@ -43,9 +43,9 @@ dependencies = [
 [[package]]
 name = "gccjit_sys"
 version = "0.0.1"
-source = "git+https://github.com/antoyo/gccjit.rs#bdb86fb5092895ff5589726b33250010c64d93f6"
+source = "git+https://github.com/antoyo/gccjit.rs#fe242b7eb26980e6c78859d51c8d4cc1e43381a3"
 dependencies = [
- "libc 0.1.12",
+ "libc",
 ]
 
 [[package]]
@@ -64,7 +64,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
 dependencies = [
  "cfg-if",
- "libc 0.2.112",
+ "libc",
  "wasi",
 ]
 
@@ -74,7 +74,7 @@ version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
 dependencies = [
- "libc 0.2.112",
+ "libc",
 ]
 
 [[package]]
@@ -85,7 +85,7 @@ checksum = "96bd995a092cac79868250589869b5a5d656b02a02bd74c8ebdc566dc7203090"
 dependencies = [
  "fm",
  "getopts",
- "libc 0.2.112",
+ "libc",
  "num_cpus",
  "termcolor",
  "threadpool",
@@ -95,12 +95,6 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122"
-
-[[package]]
-name = "libc"
 version = "0.2.112"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
@@ -118,7 +112,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
 dependencies = [
  "hermit-abi",
- "libc 0.2.112",
+ "libc",
 ]
 
 [[package]]
@@ -133,7 +127,7 @@ version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
 dependencies = [
- "libc 0.2.112",
+ "libc",
  "rand_chacha",
  "rand_core",
  "rand_hc",
@@ -208,6 +202,7 @@ version = "0.1.0"
 dependencies = [
  "gccjit",
  "lang_tester",
+ "smallvec",
  "tempfile",
 ]
 
@@ -221,13 +216,19 @@ dependencies = [
 ]
 
 [[package]]
+name = "smallvec"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+
+[[package]]
 name = "tempfile"
 version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
 dependencies = [
  "cfg-if",
- "libc 0.2.112",
+ "libc",
  "rand",
  "redox_syscall",
  "remove_dir_all",
@@ -264,7 +265,7 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
 dependencies = [
- "libc 0.2.112",
+ "libc",
 ]
 
 [[package]]
diff --git a/compiler/rustc_codegen_gcc/Cargo.toml b/compiler/rustc_codegen_gcc/Cargo.toml
index 1f3da2f79..81066d9ce 100644
--- a/compiler/rustc_codegen_gcc/Cargo.toml
+++ b/compiler/rustc_codegen_gcc/Cargo.toml
@@ -27,6 +27,8 @@ gccjit = { git = "https://github.com/antoyo/gccjit.rs" }
 # Local copy.
 #gccjit = { path = "../gccjit.rs" }
 
+smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
+
 [dev-dependencies]
 lang_tester = "0.3.9"
 tempfile = "3.1.0"
diff --git a/compiler/rustc_codegen_gcc/Readme.md b/compiler/rustc_codegen_gcc/Readme.md
index fe23a2676..bb7419438 100644
--- a/compiler/rustc_codegen_gcc/Readme.md
+++ b/compiler/rustc_codegen_gcc/Readme.md
@@ -1,5 +1,7 @@
 # WIP libgccjit codegen backend for rust
 
+[![Chat on IRC](https://img.shields.io/badge/irc.libera.chat-%23rustc__codegen__gcc-blue.svg)](https://web.libera.chat/#rustc_codegen_gcc)
+
 This is a GCC codegen for rustc, which means it can be loaded by the existing rustc frontend, but benefits from GCC: more architectures are supported and GCC's optimizations are used.
 
 **Despite its name, libgccjit can be used for ahead-of-time compilation, as is used here.**
@@ -16,21 +18,61 @@ The patches in [this repository](https://github.com/antoyo/libgccjit-patches) ne
 (Those patches should work when applied on master, but in case it doesn't work, they are known to work when applied on 079c23cfe079f203d5df83fea8e92a60c7d7e878.)
 You can also use my [fork of gcc](https://github.com/antoyo/gcc) which already includes these patches.**
 
+To build it (most of these instructions come from [here](https://gcc.gnu.org/onlinedocs/jit/internals/index.html), so don't hesitate to take a look there if you encounter an issue):
+
+```bash
+$ git clone https://github.com/antoyo/gcc
+$ sudo apt install flex libmpfr-dev libgmp-dev libmpc3 libmpc-dev
+$ mkdir gcc-build gcc-install
+$ cd gcc-build
+$ ../gcc/configure \
+    --enable-host-shared \
+    --enable-languages=jit \
+    --enable-checking=release \ # it enables extra checks which allow to find bugs
+    --disable-bootstrap \
+    --disable-multilib \
+    --prefix=$(pwd)/../gcc-install
+$ make -j4 # You can replace `4` with another number depending on how many cores you have.
+```
+
+If you want to run libgccjit tests, you will need to also enable the C++ language in the `configure`:
+
+```bash
+--enable-languages=jit,c++
+```
+
+Then to run libgccjit tests:
+
+```bash
+$ cd gcc # from the `gcc-build` folder
+$ make check-jit
+# To run one specific test:
+$ make check-jit RUNTESTFLAGS="-v -v -v jit.exp=jit.dg/test-asm.cc"
+```
+
 **Put the path to your custom build of libgccjit in the file `gcc_path`.**
 
 ```bash
-$ git clone https://github.com/rust-lang/rustc_codegen_gcc.git
-$ cd rustc_codegen_gcc
+$ dirname $(readlink -f `find . -name libgccjit.so`) > gcc_path
+```
+
+You also need to set RUST_COMPILER_RT_ROOT:
+
+```bash
 $ git clone https://github.com/llvm/llvm-project llvm --depth 1 --single-branch
 $ export RUST_COMPILER_RT_ROOT="$PWD/llvm/compiler-rt"
-$ ./prepare_build.sh # download and patch sysroot src
-$ ./build.sh --release
 ```
 
-To run the tests:
+Then you can run commands like this:
 
 ```bash
 $ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking
+$ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./build.sh --release
+```
+
+To run the tests:
+
+```bash
 $ ./test.sh --release
 ```
 
@@ -120,13 +162,52 @@ To print a debug representation of a tree:
 debug_tree(expr);
 ```
 
+(defined in print-tree.h)
+
+To print a debug reprensentation of a gimple struct:
+
+```c
+debug_gimple_stmt(gimple_struct)
+```
+
 To get the `rustc` command to run in `gdb`, add the `--verbose` flag to `cargo build`.
 
+To have the correct file paths in `gdb` instead of `/usr/src/debug/gcc/libstdc++-v3/libsupc++/eh_personality.cc`:
+
+Maybe by calling the following at the beginning of gdb:
+
+```
+set substitute-path /usr/src/debug/gcc /path/to/gcc-repo/gcc
+```
+
+TODO(antoyo): but that's not what I remember I was doing.
+
 ### How to use a custom-build rustc
 
  * Build the stage2 compiler (`rustup toolchain link debug-current build/x86_64-unknown-linux-gnu/stage2`).
  * Clean and rebuild the codegen with `debug-current` in the file `rust-toolchain`.
 
+### How to install a forked git-subtree
+
+Using git-subtree with `rustc` requires a patched git to make it work.
+The PR that is needed is [here](https://github.com/gitgitgadget/git/pull/493).
+Use the following instructions to install it:
+
+```
+git clone git@github.com:tqc/git.git
+cd git
+git checkout tqc/subtree
+make
+make install
+cd contrib/subtree
+make
+cp git-subtree ~/bin
+```
+
+### How to use [mem-trace](https://github.com/antoyo/mem-trace)
+
+`rustc` needs to be built without `jemalloc` so that `mem-trace` can overload `malloc` since `jemalloc` is linked statically, so a `LD_PRELOAD`-ed library won't a chance to intercept the calls to `malloc`.
+
 ### How to build a cross-compiling libgccjit
 
 #### Building libgccjit
@@ -142,6 +223,5 @@ To get the `rustc` command to run in `gdb`, add the `--verbose` flag to `cargo b
  * Since rustc doesn't support this architecture yet, set it back to `TARGET_TRIPLE="mips-unknown-linux-gnu"` (or another target having the same attributes). Alternatively, create a [target specification file](https://book.avr-rust.com/005.1-the-target-specification-json-file.html) (note that the `arch` specified in this file must be supported by the rust compiler).
  * Set `linker='-Clinker=m68k-linux-gcc'`.
  * Set the path to the cross-compiling libgccjit in `gcc_path`.
- * Disable the 128-bit integer types if the target doesn't support them by using `let i128_type = context.new_type::<i64>();` in `context.rs` (same for u128_type).
  * Comment the line: `context.add_command_line_option("-masm=intel");` in src/base.rs.
  * (might not be necessary) Disable the compilation of libstd.so (and possibly libcore.so?).
diff --git a/compiler/rustc_codegen_gcc/build_sysroot/build_sysroot.sh b/compiler/rustc_codegen_gcc/build_sysroot/build_sysroot.sh
index f293192a0..9d692d599 100755
--- a/compiler/rustc_codegen_gcc/build_sysroot/build_sysroot.sh
+++ b/compiler/rustc_codegen_gcc/build_sysroot/build_sysroot.sh
@@ -16,7 +16,7 @@ rm Cargo.lock test_target/Cargo.lock 2>/dev/null || true
 rm -r sysroot/ 2>/dev/null || true
 
 # Build libs
-export RUSTFLAGS="$RUSTFLAGS -Z force-unstable-if-unmarked -Cpanic=abort"
+export RUSTFLAGS="$RUSTFLAGS -Z force-unstable-if-unmarked"
 if [[ "$1" == "--release" ]]; then
     sysroot_channel='release'
     RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target $TARGET_TRIPLE --release
diff --git a/compiler/rustc_codegen_gcc/config.sh b/compiler/rustc_codegen_gcc/config.sh
index b25e215fb..166e83901 100644
--- a/compiler/rustc_codegen_gcc/config.sh
+++ b/compiler/rustc_codegen_gcc/config.sh
@@ -38,7 +38,7 @@ if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
    fi
 fi
 
-export RUSTFLAGS="$CG_RUSTFLAGS $linker -Cpanic=abort -Csymbol-mangling-version=v0 -Cdebuginfo=2 -Clto=off -Zpanic-abort-tests -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot"
+export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 -Clto=off -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS"
 
 # FIXME(antoyo): remove once the atomic shim is gone
 if [[ `uname` == 'Darwin' ]]; then
diff --git a/compiler/rustc_codegen_gcc/example/alloc_example.rs b/compiler/rustc_codegen_gcc/example/alloc_example.rs
index 74ea7ec4e..754e79314 100644
--- a/compiler/rustc_codegen_gcc/example/alloc_example.rs
+++ b/compiler/rustc_codegen_gcc/example/alloc_example.rs
@@ -1,4 +1,4 @@
-#![feature(start, box_syntax, core_intrinsics, alloc_error_handler)]
+#![feature(start, core_intrinsics, alloc_error_handler, lang_items)]
 #![no_std]
 
 extern crate alloc;
@@ -18,21 +18,27 @@ extern "C" {
 
 #[panic_handler]
 fn panic_handler(_: &core::panic::PanicInfo) -> ! {
-    unsafe {
-        core::intrinsics::abort();
-    }
+    core::intrinsics::abort();
 }
 
 #[alloc_error_handler]
 fn alloc_error_handler(_: alloc::alloc::Layout) -> ! {
-    unsafe {
-        core::intrinsics::abort();
-    }
+    core::intrinsics::abort();
+}
+
+#[lang = "eh_personality"]
+fn eh_personality() -> ! {
+    loop {}
+}
+
+#[no_mangle]
+unsafe extern "C" fn _Unwind_Resume() {
+    core::intrinsics::unreachable();
 }
 
 #[start]
 fn main(_argc: isize, _argv: *const *const u8) -> isize {
-    let world: Box<&str> = box "Hello World!\0";
+    let world: Box<&str> = Box::new("Hello World!\0");
     unsafe {
         puts(*world as *const str as *const u8);
     }
diff --git a/compiler/rustc_codegen_gcc/example/alloc_system.rs b/compiler/rustc_codegen_gcc/example/alloc_system.rs
index fd01fcf1f..046903fe5 100644
--- a/compiler/rustc_codegen_gcc/example/alloc_system.rs
+++ b/compiler/rustc_codegen_gcc/example/alloc_system.rs
@@ -1,12 +1,6 @@
-// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
+// SPDX-License-Identifier: MIT OR Apache-2.0
+// SPDX-FileCopyrightText: The Rust Project Developers (see https://thanks.rust-lang.org)
+
 #![no_std]
 #![feature(allocator_api, rustc_private)]
 #![cfg_attr(any(unix, target_os = "redox"), feature(libc))]
@@ -21,6 +15,7 @@
 const MIN_ALIGN: usize = 8;
 #[cfg(any(target_arch = "x86_64",
               target_arch = "aarch64",
+              target_arch = "loongarch64",
               target_arch = "mips64",
               target_arch = "s390x",
               target_arch = "sparc64"))]
diff --git a/compiler/rustc_codegen_gcc/example/mini_core.rs b/compiler/rustc_codegen_gcc/example/mini_core.rs
index ddcbb0d9f..637b8dc53 100644
--- a/compiler/rustc_codegen_gcc/example/mini_core.rs
+++ b/compiler/rustc_codegen_gcc/example/mini_core.rs
@@ -1,6 +1,6 @@
 #![feature(
     no_core, lang_items, intrinsics, unboxed_closures, type_ascription, extern_types,
-    untagged_unions, decl_macro, rustc_attrs, transparent_unions, auto_traits,
+    decl_macro, rustc_attrs, transparent_unions, auto_traits,
     thread_local
 )]
 #![no_core]
@@ -17,6 +17,9 @@ pub trait Sized {}
 #[lang = "destruct"]
 pub trait Destruct {}
 
+#[lang = "tuple_trait"]
+pub trait Tuple {}
+
 #[lang = "unsize"]
 pub trait Unsize<T: ?Sized> {}
 
@@ -39,14 +42,14 @@ impl<'a, T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<&'a mut U> for &'a mut
 impl<T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<*const U> for *const T {}
 // *mut T -> *mut U
 impl<T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<*mut U> for *mut T {}
-impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<Box<U>> for Box<T> {}
+impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<Box<U, ()>> for Box<T, ()> {}
 
 #[lang = "receiver"]
 pub trait Receiver {}
 
 impl<T: ?Sized> Receiver for &T {}
 impl<T: ?Sized> Receiver for &mut T {}
-impl<T: ?Sized> Receiver for Box<T> {}
+impl<T: ?Sized, A: Allocator> Receiver for Box<T, A> {}
 
 #[lang = "copy"]
 pub unsafe trait Copy {}
@@ -396,7 +399,7 @@ pub struct PhantomData<T: ?Sized>;
 
 #[lang = "fn_once"]
 #[rustc_paren_sugar]
-pub trait FnOnce<Args> {
+pub trait FnOnce<Args: Tuple> {
     #[lang = "fn_once_output"]
     type Output;
 
@@ -405,13 +408,21 @@ pub trait FnOnce<Args> {
 
 #[lang = "fn_mut"]
 #[rustc_paren_sugar]
-pub trait FnMut<Args>: FnOnce<Args> {
+pub trait FnMut<Args: Tuple>: FnOnce<Args> {
     extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output;
 }
 
 #[lang = "panic"]
 #[track_caller]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
+    unsafe {
+        libc::puts("Panicking\n\0" as *const str as *const u8);
+        intrinsics::abort();
+    }
+}
+
+#[lang = "panic_cannot_unwind"]
+fn panic_cannot_unwind() -> ! {
     unsafe {
         libc::puts("Panicking\n\0" as *const str as *const u8);
         intrinsics::abort();
@@ -450,17 +461,32 @@ pub trait Deref {
 pub trait Allocator {
 }
 
+impl Allocator for () {}
+
 pub struct Global;
 
 impl Allocator for Global {}
 
+#[repr(transparent)]
+#[rustc_layout_scalar_valid_range_start(1)]
+#[rustc_nonnull_optimization_guaranteed]
+pub struct NonNull<T: ?Sized>(pub *const T);
+
+impl<T: ?Sized, U: ?Sized> CoerceUnsized<NonNull<U>> for NonNull<T> where T: Unsize<U> {}
+impl<T: ?Sized, U: ?Sized> DispatchFromDyn<NonNull<U>> for NonNull<T> where T: Unsize<U> {}
+
+pub struct Unique<T: ?Sized> {
+    pub pointer: NonNull<T>,
+    pub _marker: PhantomData<T>,
+}
+
+impl<T: ?Sized, U: ?Sized> CoerceUnsized<Unique<U>> for Unique<T> where T: Unsize<U> {}
+impl<T: ?Sized, U: ?Sized> DispatchFromDyn<Unique<U>> for Unique<T> where T: Unsize<U> {}
+
 #[lang = "owned_box"]
-pub struct Box<
-    T: ?Sized,
-    A: Allocator = Global,
->(*mut T, A);
+pub struct Box<T: ?Sized, A: Allocator = Global>(Unique<T>, A);
 
-impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<Box<U>> for Box<T> {}
+impl<T: ?Sized + Unsize<U>, U: ?Sized, A: Allocator> CoerceUnsized<Box<U, A>> for Box<T, A> {}
 
 impl<T: ?Sized, A: Allocator> Drop for Box<T, A> {
     fn drop(&mut self) {
@@ -468,7 +494,7 @@ impl<T: ?Sized, A: Allocator> Drop for Box<T, A> {
     }
 }
 
-impl<T> Deref for Box<T> {
+impl<T: ?Sized, A: Allocator> Deref for Box<T, A> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -482,8 +508,8 @@ unsafe fn allocate(size: usize, _align: usize) -> *mut u8 {
 }
 
 #[lang = "box_free"]
-unsafe fn box_free<T: ?Sized, A: Allocator>(ptr: *mut T, alloc: A) {
-    libc::free(ptr as *mut u8);
+unsafe fn box_free<T: ?Sized>(ptr: Unique<T>, _alloc: ()) {
+    libc::free(ptr.pointer.0 as *mut u8);
 }
 
 #[lang = "drop"]
@@ -505,17 +531,25 @@ pub union MaybeUninit<T> {
 }
 
 pub mod intrinsics {
+    use crate::Sized;
+
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
+        #[rustc_safe_intrinsic]
         pub fn size_of<T>() -> usize;
-        pub fn size_of_val<T: ?::Sized>(val: *const T) -> usize;
+        pub fn size_of_val<T: ?Sized>(val: *const T) -> usize;
+        #[rustc_safe_intrinsic]
         pub fn min_align_of<T>() -> usize;
-        pub fn min_align_of_val<T: ?::Sized>(val: *const T) -> usize;
+        pub fn min_align_of_val<T: ?Sized>(val: *const T) -> usize;
         pub fn copy<T>(src: *const T, dst: *mut T, count: usize);
         pub fn transmute<T, U>(e: T) -> U;
         pub fn ctlz_nonzero<T>(x: T) -> T;
-        pub fn needs_drop<T: ?::Sized>() -> bool;
+        #[rustc_safe_intrinsic]
+        pub fn needs_drop<T: ?Sized>() -> bool;
+        #[rustc_safe_intrinsic]
         pub fn bitreverse<T>(x: T) -> T;
+        #[rustc_safe_intrinsic]
         pub fn bswap<T>(x: T) -> T;
         pub fn write_bytes<T>(dst: *mut T, val: u8, count: usize);
         pub fn unreachable() -> !;
diff --git a/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs b/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs
index 14fd9eeff..cff260777 100644
--- a/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs
+++ b/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs
@@ -1,7 +1,7 @@
 // Adapted from https://github.com/sunfishcode/mir2cranelift/blob/master/rust-examples/nocore-hello-world.rs
 
 #![feature(
-    no_core, unboxed_closures, start, lang_items, box_syntax, never_type, linkage,
+    no_core, unboxed_closures, start, lang_items, never_type, linkage,
     extern_types, thread_local
 )]
 #![no_core]
@@ -85,6 +85,7 @@ fn start<T: Termination + 'static>(
     main: fn() -> T,
     argc: isize,
     argv: *const *const u8,
+    _sigpipe: u8,
 ) -> isize {
     if argc == 3 {
         unsafe { puts(*argv); }
@@ -162,7 +163,7 @@ fn main() {
         let ptr: *const u8 = hello as *const [u8] as *const u8;
         puts(ptr);
 
-        let world: Box<&str> = box "World!\0";
+        let world: Box<&str> = Box::new("World!\0");
         puts(*world as *const str as *const u8);
         world as Box<dyn SomeTrait>;
 
@@ -222,12 +223,13 @@ fn main() {
         }
     }
 
-    let _ = box NoisyDrop {
+    let _ = Box::new(NoisyDrop {
         text: "Boxed outer got dropped!\0",
         inner: NoisyDropInner,
-    } as Box<dyn SomeTrait>;
+    }) as Box<dyn SomeTrait>;
 
     const FUNC_REF: Option<fn()> = Some(main);
+    #[allow(unreachable_code)]
     match FUNC_REF {
         Some(_) => {},
         None => assert!(false),
diff --git a/compiler/rustc_codegen_gcc/example/mod_bench.rs b/compiler/rustc_codegen_gcc/example/mod_bench.rs
index 2e2b0052d..5e2e7f25a 100644
--- a/compiler/rustc_codegen_gcc/example/mod_bench.rs
+++ b/compiler/rustc_codegen_gcc/example/mod_bench.rs
@@ -1,4 +1,4 @@
-#![feature(start, box_syntax, core_intrinsics, lang_items)]
+#![feature(start, core_intrinsics, lang_items)]
 #![no_std]
 
 #[link(name = "c")]
@@ -6,9 +6,7 @@ extern {}
 
 #[panic_handler]
 fn panic_handler(_: &core::panic::PanicInfo) -> ! {
-    unsafe {
-        core::intrinsics::abort();
-    }
+    core::intrinsics::abort();
 }
 
 #[lang="eh_personality"]
@@ -32,6 +30,6 @@ fn main(_argc: isize, _argv: *const *const u8) -> isize {
 #[inline(never)]
 fn black_box(i: u32) {
     if i != 1 {
-        unsafe { core::intrinsics::abort(); }
+        core::intrinsics::abort();
     }
 }
diff --git a/compiler/rustc_codegen_gcc/example/std_example.rs b/compiler/rustc_codegen_gcc/example/std_example.rs
index 31069058a..5c171c49f 100644
--- a/compiler/rustc_codegen_gcc/example/std_example.rs
+++ b/compiler/rustc_codegen_gcc/example/std_example.rs
@@ -1,5 +1,6 @@
 #![feature(core_intrinsics, generators, generator_trait, is_sorted)]
 
+#[cfg(feature="master")]
 use std::arch::x86_64::*;
 use std::io::Write;
 use std::ops::Generator;
diff --git a/compiler/rustc_codegen_gcc/failing-ui-tests.txt b/compiler/rustc_codegen_gcc/failing-ui-tests.txt
new file mode 100644
index 000000000..8539e27ea
--- /dev/null
+++ b/compiler/rustc_codegen_gcc/failing-ui-tests.txt
@@ -0,0 +1,68 @@
+tests/ui/allocator/custom-in-block.rs
+tests/ui/allocator/custom-in-submodule.rs
+tests/ui/allocator/custom.rs
+tests/ui/allocator/hygiene.rs
+tests/ui/allocator/no_std-alloc-error-handler-custom.rs
+tests/ui/allocator/no_std-alloc-error-handler-default.rs
+tests/ui/allocator/xcrate-use.rs
+tests/ui/allocator/xcrate-use2.rs
+tests/ui/asm/may_unwind.rs
+tests/ui/asm/x86_64/multiple-clobber-abi.rs
+tests/ui/debuginfo/debuginfo-emit-llvm-ir-and-split-debuginfo.rs
+tests/ui/functions-closures/parallel-codegen-closures.rs
+tests/ui/linkage-attr/linkage1.rs
+tests/ui/lto/dylib-works.rs
+tests/ui/numbers-arithmetic/saturating-float-casts.rs
+tests/ui/polymorphization/promoted-function.rs
+tests/ui/process/nofile-limit.rs
+tests/ui/sepcomp/sepcomp-cci.rs
+tests/ui/sepcomp/sepcomp-extern.rs
+tests/ui/sepcomp/sepcomp-fns-backwards.rs
+tests/ui/sepcomp/sepcomp-fns.rs
+tests/ui/sepcomp/sepcomp-statics.rs
+tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
+tests/ui/sse2.rs
+tests/ui/target-feature/missing-plusminus.rs
+tests/ui/asm/x86_64/may_unwind.rs
+tests/ui/backtrace.rs
+tests/ui/catch-unwind-bang.rs
+tests/ui/cfg/cfg-panic-abort.rs
+tests/ui/drop/dynamic-drop-async.rs
+tests/ui/drop/repeat-drop.rs
+tests/ui/fmt/format-args-capture.rs
+tests/ui/generator/panic-drops-resume.rs
+tests/ui/generator/panic-drops.rs
+tests/ui/intrinsics/panic-uninitialized-zeroed.rs
+tests/ui/iterators/iter-sum-overflow-debug.rs
+tests/ui/iterators/iter-sum-overflow-overflow-checks.rs
+tests/ui/mir/mir_calls_to_shims.rs
+tests/ui/mir/mir_drop_order.rs
+tests/ui/mir/mir_let_chains_drop_order.rs
+tests/ui/oom_unwind.rs
+tests/ui/panic-runtime/abort-link-to-unwinding-crates.rs
+tests/ui/panic-runtime/abort.rs
+tests/ui/panic-runtime/link-to-abort.rs
+tests/ui/unwind-no-uwtable.rs
+tests/ui/parser/unclosed-delimiter-in-dep.rs
+tests/ui/runtime/rt-explody-panic-payloads.rs
+tests/ui/simd/intrinsic/ptr-cast.rs
+tests/ui/binding/fn-arg-incomplete-pattern-drop-order.rs
+tests/ui/consts/missing_span_in_backtrace.rs
+tests/ui/drop/dynamic-drop.rs
+tests/ui/dyn-star/box.rs
+tests/ui/issues/issue-40883.rs
+tests/ui/issues/issue-43853.rs
+tests/ui/issues/issue-47364.rs
+tests/ui/macros/rfc-2011-nicer-assert-messages/assert-without-captures-does-not-create-unnecessary-code.rs
+tests/ui/rfc-2091-track-caller/std-panic-locations.rs
+tests/ui/rfcs/rfc1857-drop-order.rs
+tests/ui/simd/issue-17170.rs
+tests/ui/simd/issue-39720.rs
+tests/ui/simd/issue-89193.rs
+tests/ui/statics/issue-91050-1.rs
+tests/ui/statics/issue-91050-2.rs
+tests/ui/alloc-error/default-alloc-error-hook.rs
+tests/ui/generator/panic-safe.rs
+tests/ui/issues/issue-14875.rs
+tests/ui/issues/issue-29948.rs
+tests/ui/panic-while-printing.rs
diff --git a/compiler/rustc_codegen_gcc/failing-ui-tests12.txt b/compiler/rustc_codegen_gcc/failing-ui-tests12.txt
new file mode 100644
index 000000000..8c27bd8b8
--- /dev/null
+++ b/compiler/rustc_codegen_gcc/failing-ui-tests12.txt
@@ -0,0 +1,39 @@
+tests/ui/asm/x86_64/issue-96797.rs
+tests/ui/intrinsics/const-eval-select-x86_64.rs
+tests/ui/packed/packed-struct-drop-aligned.rs
+tests/ui/packed/packed-struct-generic-layout.rs
+tests/ui/packed/packed-struct-layout.rs
+tests/ui/packed/packed-struct-optimized-enum.rs
+tests/ui/packed/packed-struct-size.rs
+tests/ui/packed/packed-struct-vec.rs
+tests/ui/packed/packed-tuple-struct-layout.rs
+tests/ui/simd/array-type.rs
+tests/ui/simd/intrinsic/float-minmax-pass.rs
+tests/ui/simd/intrinsic/generic-arithmetic-saturating-pass.rs
+tests/ui/simd/intrinsic/generic-as.rs
+tests/ui/simd/intrinsic/generic-cast-pass.rs
+tests/ui/simd/intrinsic/generic-cast-pointer-width.rs
+tests/ui/simd/intrinsic/generic-comparison-pass.rs
+tests/ui/simd/intrinsic/generic-elements-pass.rs
+tests/ui/simd/intrinsic/generic-reduction-pass.rs
+tests/ui/simd/intrinsic/generic-select-pass.rs
+tests/ui/simd/intrinsic/inlining-issue67557-ice.rs
+tests/ui/simd/intrinsic/inlining-issue67557.rs
+tests/ui/simd/monomorphize-shuffle-index.rs
+tests/ui/simd/shuffle.rs
+tests/ui/simd/simd-bitmask.rs
+tests/ui/generator/resume-after-return.rs
+tests/ui/iterators/iter-step-overflow-debug.rs
+tests/ui/macros/rfc-2011-nicer-assert-messages/all-expr-kinds.rs
+tests/ui/numbers-arithmetic/next-power-of-two-overflow-debug.rs
+tests/ui/privacy/reachable-unnameable-items.rs
+tests/ui/rfc-1937-termination-trait/termination-trait-in-test.rs
+tests/ui/async-await/async-fn-size-moved-locals.rs
+tests/ui/async-await/async-fn-size-uninit-locals.rs
+tests/ui/cfg/cfg-panic.rs
+tests/ui/generator/size-moved-locals.rs
+tests/ui/macros/rfc-2011-nicer-assert-messages/all-not-available-cases.rs
+tests/ui/simd/intrinsic/generic-gather-pass.rs
+tests/ui/simd/issue-85915-simd-ptrs.rs
+tests/ui/issues/issue-68010-large-zst-consts.rs
+tests/ui/rust-2018/proc-macro-crate-in-paths.rs
diff --git a/compiler/rustc_codegen_gcc/locales/en-US.ftl b/compiler/rustc_codegen_gcc/messages.ftl
index 6101b28ab..0a94a08f8 100644
--- a/compiler/rustc_codegen_gcc/locales/en-US.ftl
+++ b/compiler/rustc_codegen_gcc/messages.ftl
@@ -60,3 +60,9 @@ codegen_gcc_invalid_monomorphization_unsupported_cast =
 
 codegen_gcc_invalid_monomorphization_unsupported_operation =
     invalid monomorphization of `{$name}` intrinsic: unsupported operation on `{$in_ty}` with element `{$in_elem}`
+
+codegen_gcc_invalid_minimum_alignment =
+    invalid minimum global alignment: {$err}
+
+codegen_gcc_tied_target_features = the target features {$features} must all be either enabled or disabled together
+    .help = add the missing features in a `target_feature` attribute
diff --git a/compiler/rustc_codegen_gcc/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch b/compiler/rustc_codegen_gcc/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch
new file mode 100644
index 000000000..93c63b5dc
--- /dev/null
+++ b/compiler/rustc_codegen_gcc/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch
@@ -0,0 +1,39 @@
+From c3821e02fbd6cb5ad6e06d759fccdc9073712375 Mon Sep 17 00:00:00 2001
+From: Antoni Boucher <bouanto@zoho.com>
+Date: Tue, 7 Jun 2022 21:40:13 -0400
+Subject: [PATCH] Add stdarch Cargo.toml for testing
+
+---
+ library/stdarch/Cargo.toml | 20 ++++++++++++++++++++
+ 1 file changed, 20 insertions(+)
+ create mode 100644 library/stdarch/Cargo.toml
+
+diff --git a/library/stdarch/Cargo.toml b/library/stdarch/Cargo.toml
+new file mode 100644
+index 0000000..fbe0a95
+--- /dev/null
++++ b/library/stdarch/Cargo.toml
+@@ -0,0 +1,20 @@
++[workspace]
++members = [
++  "crates/core_arch",
++  "crates/std_detect",
++  "crates/stdarch-gen",
++  "examples/"
++]
++exclude = [
++  "crates/wasm-assert-instr-tests"
++]
++
++[profile.release]
++debug = true
++opt-level = 3
++incremental = true
++
++[profile.bench]
++debug = 1
++opt-level = 3
++incremental = true
+-- 
+2.26.2.7.g19db9cfb68.dirty
+
diff --git a/compiler/rustc_codegen_gcc/patches/0001-Disable-examples.patch b/compiler/rustc_codegen_gcc/patches/0001-Disable-examples.patch
new file mode 100644
index 000000000..1b71df1ca
--- /dev/null
+++ b/compiler/rustc_codegen_gcc/patches/0001-Disable-examples.patch
@@ -0,0 +1,25 @@
+From a2d53a324a02c04b76c0e9d39dc15cd443a3b8b2 Mon Sep 17 00:00:00 2001
+From: Antoni Boucher <bouanto@zoho.com>
+Date: Fri, 25 Nov 2022 11:18:11 -0500
+Subject: [PATCH] Disable examples
+
+---
+ library/stdarch/Cargo.toml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/library/stdarch/Cargo.toml b/library/stdarch/Cargo.toml
+index fbe0a95..748d72d 100644
+--- a/library/stdarch/Cargo.toml
++++ b/library/stdarch/Cargo.toml
+@@ -3,7 +3,7 @@ members = [
+   "crates/core_arch",
+   "crates/std_detect",
+   "crates/stdarch-gen",
+-  "examples/"
++  #"examples/"
+ ]
+ exclude = [
+   "crates/wasm-assert-instr-tests"
+-- 
+2.26.2.7.g19db9cfb68.dirty
+
diff --git a/compiler/rustc_codegen_gcc/patches/0022-core-Disable-not-compiling-tests.patch b/compiler/rustc_codegen_gcc/patches/0022-core-Disable-not-compiling-tests.patch
index 301b3f9bd..4db56fa3b 100644
--- a/compiler/rustc_codegen_gcc/patches/0022-core-Disable-not-compiling-tests.patch
+++ b/compiler/rustc_codegen_gcc/patches/0022-core-Disable-not-compiling-tests.patch
@@ -18,7 +18,7 @@ new file mode 100644
 index 0000000..46fd999
 --- /dev/null
 +++ b/library/core/tests/Cargo.toml
-@@ -0,0 +1,8 @@
+@@ -0,0 +1,12 @@
 +[package]
 +name = "core"
 +version = "0.0.0"
@@ -27,37 +27,18 @@ index 0000000..46fd999
 +[lib]
 +name = "coretests"
 +path = "lib.rs"
-diff --git a/library/core/tests/num/flt2dec/mod.rs b/library/core/tests/num/flt2dec/mod.rs
-index a35897e..f0bf645 100644
---- a/library/core/tests/num/flt2dec/mod.rs
-+++ b/library/core/tests/num/flt2dec/mod.rs
-@@ -13,7 +13,6 @@ mod strategy {
-     mod dragon;
-     mod grisu;
- }
--mod random;
- 
- pub fn decode_finite<T: DecodableFloat>(v: T) -> Decoded {
-     match decode(v).1 {
-diff --git a/library/core/tests/slice.rs b/library/core/tests/slice.rs
-index 6609bc3..241b497 100644
---- a/library/core/tests/slice.rs
-+++ b/library/core/tests/slice.rs
-@@ -1209,6 +1209,7 @@ fn brute_force_rotate_test_1() {
-     }
- }
- 
-+/*
- #[test]
- #[cfg(not(target_arch = "wasm32"))]
- fn sort_unstable() {
-@@ -1394,6 +1395,7 @@ fn partition_at_index() {
-     v.select_nth_unstable(0);
-     assert!(v == [0xDEADBEEF]);
- }
-+*/
- 
- #[test]
- #[should_panic(expected = "index 0 greater than length of slice")]
++
++[dependencies]
++rand = { version = "0.8.5", default-features = false }
++rand_xorshift = { version = "0.3.0", default-features = false }
+diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
+index 42a26ae..5ac1042 100644
+--- a/library/core/tests/lib.rs
++++ b/library/core/tests/lib.rs
+@@ -1,3 +1,4 @@
++#![cfg(test)]
+ #![feature(alloc_layout_extra)]
+ #![feature(array_chunks)]
+ #![feature(array_methods)]
 --
 2.21.0 (Apple Git-122)
diff --git a/compiler/rustc_codegen_gcc/patches/0024-core-Disable-portable-simd-test.patch b/compiler/rustc_codegen_gcc/patches/0024-core-Disable-portable-simd-test.patch
deleted file mode 100644
index c59a40df0..000000000
--- a/compiler/rustc_codegen_gcc/patches/0024-core-Disable-portable-simd-test.patch
+++ /dev/null
@@ -1,28 +0,0 @@
-From b1ae000f6da1abd3b8e9b80c40bc11c89b8ae93c Mon Sep 17 00:00:00 2001
-From: bjorn3 <bjorn3@users.noreply.github.com>
-Date: Thu, 30 Dec 2021 16:54:40 +0100
-Subject: [PATCH] [core] Disable portable-simd test
-
----
- library/core/tests/lib.rs | 1 -
- 1 file changed, 1 deletion(-)
-
-diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
-index 06c7be0..359e2e7 100644
---- a/library/core/tests/lib.rs
-+++ b/library/core/tests/lib.rs
-@@ -75,7 +75,6 @@
- #![feature(never_type)]
- #![feature(unwrap_infallible)]
--#![feature(portable_simd)]
- #![feature(ptr_metadata)]
- #![feature(once_cell)]
- #![feature(option_result_contains)]
-@@ -127,7 +126,6 @@ mod pin;
- mod pin_macro;
- mod ptr;
- mod result;
--mod simd;
- mod slice;
- mod str;
- mod str_lossy;
diff --git a/compiler/rustc_codegen_gcc/rust-toolchain b/compiler/rustc_codegen_gcc/rust-toolchain
index b20aeb979..933ecd45b 100644
--- a/compiler/rustc_codegen_gcc/rust-toolchain
+++ b/compiler/rustc_codegen_gcc/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-06-06"
+channel = "nightly-2023-03-02"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/compiler/rustc_codegen_gcc/rustc_patches/compile_test.patch b/compiler/rustc_codegen_gcc/rustc_patches/compile_test.patch
deleted file mode 100644
index 59143eac3..000000000
--- a/compiler/rustc_codegen_gcc/rustc_patches/compile_test.patch
+++ /dev/null
@@ -1,14 +0,0 @@
-diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs
-index 887d27fd6dca4..2c2239f2b83d1 100644
---- a/src/tools/compiletest/src/header.rs
-+++ b/src/tools/compiletest/src/header.rs
-@@ -806,8 +806,8 @@ pub fn make_test_description<R: Read>(
-     cfg: Option<&str>,
- ) -> test::TestDesc {
-     let mut ignore = false;
-     #[cfg(not(bootstrap))]
--    let ignore_message: Option<String> = None;
-+    let ignore_message: Option<&str> = None;
-     let mut should_fail = false;
-
-     let rustc_has_profiler_support = env::var_os("RUSTC_PROFILER_SUPPORT").is_some();
diff --git a/compiler/rustc_codegen_gcc/src/allocator.rs b/compiler/rustc_codegen_gcc/src/allocator.rs
index e2c9ffe9c..4bad33ee8 100644
--- a/compiler/rustc_codegen_gcc/src/allocator.rs
+++ b/compiler/rustc_codegen_gcc/src/allocator.rs
@@ -1,3 +1,5 @@
+#[cfg(feature="master")]
+use gccjit::FnAttribute;
 use gccjit::{FunctionType, GlobalKind, ToRValue};
 use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
 use rustc_middle::bug;
@@ -50,7 +52,8 @@ pub(crate) unsafe fn codegen(tcx: TyCtxt<'_>, mods: &mut GccContext, _module_nam
         let func = context.new_function(None, FunctionType::Exported, output.unwrap_or(void), &args, name, false);
 
         if tcx.sess.target.options.default_hidden_visibility {
-            // TODO(antoyo): set visibility.
+            #[cfg(feature="master")]
+            func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
         }
         if tcx.sess.must_emit_unwind_tables() {
             // TODO(antoyo): emit unwind tables.
@@ -61,7 +64,8 @@ pub(crate) unsafe fn codegen(tcx: TyCtxt<'_>, mods: &mut GccContext, _module_nam
             .map(|(index, typ)| context.new_parameter(None, *typ, &format!("param{}", index)))
             .collect();
         let callee = context.new_function(None, FunctionType::Extern, output.unwrap_or(void), &args, callee, false);
-        // TODO(antoyo): set visibility.
+        #[cfg(feature="master")]
+        callee.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
 
         let block = func.new_block("entry");
 
@@ -90,12 +94,18 @@ pub(crate) unsafe fn codegen(tcx: TyCtxt<'_>, mods: &mut GccContext, _module_nam
         .collect();
     let func = context.new_function(None, FunctionType::Exported, void, &args, name, false);
 
+    if tcx.sess.target.default_hidden_visibility {
+        #[cfg(feature="master")]
+        func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
+    }
+
     let callee = alloc_error_handler_kind.fn_name(sym::oom);
     let args: Vec<_> = types.iter().enumerate()
         .map(|(index, typ)| context.new_parameter(None, *typ, &format!("param{}", index)))
         .collect();
     let callee = context.new_function(None, FunctionType::Extern, void, &args, callee, false);
-    //llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden);
+    #[cfg(feature="master")]
+    callee.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
 
     let block = func.new_block("entry");
 
diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs
index c346dbd63..65de02b35 100644
--- a/compiler/rustc_codegen_gcc/src/asm.rs
+++ b/compiler/rustc_codegen_gcc/src/asm.rs
@@ -157,7 +157,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     use ConstraintOrRegister::*;
 
                     let (constraint, ty) = match (reg_to_gcc(reg), place) {
-                        (Constraint(constraint), Some(place)) => (constraint, place.layout.gcc_type(self.cx, false)),
+                        (Constraint(constraint), Some(place)) => (constraint, place.layout.gcc_type(self.cx)),
                         // When `reg` is a class and not an explicit register but the out place is not specified,
                         // we need to create an unused output variable to assign the output to. This var
                         // needs to be of a type that's "compatible" with the register class, but specific type
@@ -226,7 +226,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     // This decision is also backed by the fact that LLVM needs in and out
                     // values to be of *exactly the same type*, not just "compatible".
                     // I'm not sure if GCC is so picky too, but better safe than sorry.
-                    let ty = in_value.layout.gcc_type(self.cx, false);
+                    let ty = in_value.layout.gcc_type(self.cx);
                     let tmp_var = self.current_func().new_local(None, ty, "output_register");
 
                     // If the out_place is None (i.e `inout(reg) _` syntax was used), we translate
@@ -286,7 +286,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                             continue
                         };
 
-                        let ty = out_place.layout.gcc_type(self.cx, false);
+                        let ty = out_place.layout.gcc_type(self.cx);
                         let tmp_var = self.current_func().new_local(None, ty, "output_register");
                         tmp_var.set_register_name(reg_name);
 
@@ -306,7 +306,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 // `in("explicit register") var`
                 InlineAsmOperandRef::In { reg, value } => {
                     if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) {
-                        let ty = value.layout.gcc_type(self.cx, false);
+                        let ty = value.layout.gcc_type(self.cx);
                         let reg_var = self.current_func().new_local(None, ty, "input_register");
                         reg_var.set_register_name(reg_name);
                         self.llbb().add_assignment(None, reg_var, value.immediate());
@@ -325,7 +325,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
                     if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) {
                         // See explanation in the first pass.
-                        let ty = in_value.layout.gcc_type(self.cx, false);
+                        let ty = in_value.layout.gcc_type(self.cx);
                         let tmp_var = self.current_func().new_local(None, ty, "output_register");
                         tmp_var.set_register_name(reg_name);
 
@@ -353,8 +353,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     inputs.push(AsmInOperand {
                         constraint: "X".into(),
                         rust_idx,
-                        val: self.cx.rvalue_as_function(get_fn(self.cx, instance))
-                            .get_address(None),
+                        val: get_fn(self.cx, instance).get_address(None),
                     });
                 }
 
@@ -382,15 +381,19 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         for piece in template {
             match *piece {
                 InlineAsmTemplatePiece::String(ref string) => {
-                    // TODO(@Commeownist): switch to `Iterator::intersperse` once it's stable
-                    let mut iter = string.split('%');
-                    if let Some(s) = iter.next() {
-                        template_str.push_str(s);
-                    }
-
-                    for s in iter {
-                        template_str.push_str("%%");
-                        template_str.push_str(s);
+                    for char in string.chars() {
+                        // TODO(antoyo): might also need to escape | if rustc doesn't do it.
+                        let escaped_char =
+                            match char {
+                                '%' => "%%",
+                                '{' => "%{",
+                                '}' => "%}",
+                                _ => {
+                                    template_str.push(char);
+                                    continue;
+                                },
+                            };
+                        template_str.push_str(escaped_char);
                     }
                 }
                 InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
@@ -565,39 +568,55 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
                 _ => unimplemented!(),
             }
         },
+        // They can be retrieved from https://gcc.gnu.org/onlinedocs/gcc/Machine-Constraints.html
         InlineAsmRegOrRegClass::RegClass(reg) => match reg {
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => unimplemented!(),
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => unimplemented!(),
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => unimplemented!(),
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => unimplemented!(),
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => unimplemented!(),
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
+                unreachable!("clobber-only")
+            }
+            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r",
             InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
             | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
-            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => unimplemented!(),
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16)
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16)
             | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8)
-            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => unimplemented!(),
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
-            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => unimplemented!(),
-            InlineAsmRegClass::Avr(_) => unimplemented!(),
-            InlineAsmRegClass::Bpf(_) => unimplemented!(),
-            InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => unimplemented!(),
-            InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => unimplemented!(),
-            InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => unimplemented!(),
-            InlineAsmRegClass::Msp430(_) => unimplemented!(),
-            InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => unimplemented!(),
-            InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => unimplemented!(),
-            InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => unimplemented!(),
-            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => unimplemented!(),
-            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => unimplemented!(),
-            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => unimplemented!(),
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4)
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "t",
+            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d",
+            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r",
+            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w",
+            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
+            InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w",
+            InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a",
+            InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d",
+            InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "d", // more specific than "r"
+            InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f",
+            InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r",
+            // https://github.com/gcc-mirror/gcc/blob/master/gcc/config/nvptx/nvptx.md -> look for
+            // "define_constraint".
+            InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
+            InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
+            InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
+
+            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
+            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
             InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
             | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
                 unreachable!("clobber-only")
             },
-            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => unimplemented!(),
-            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => unimplemented!(),
-            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => unimplemented!(),
+            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
+            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
+                unreachable!("clobber-only")
+            }
             InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r",
             InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q",
             InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q",
@@ -605,16 +624,18 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
             | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x",
             InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v",
             InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "Yk",
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg0) => unimplemented!(),
-            InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => unimplemented!(),
             InlineAsmRegClass::X86(
-                X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::tmm_reg,
+                X86InlineAsmRegClass::kreg0
+                | X86InlineAsmRegClass::x87_reg
+                | X86InlineAsmRegClass::mmx_reg
+                | X86InlineAsmRegClass::tmm_reg,
             ) => unreachable!("clobber-only"),
             InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
                 bug!("GCC backend does not support SPIR-V")
             }
-            InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => unimplemented!(),
-            InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => unimplemented!(),
+            InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r",
+            InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
             InlineAsmRegClass::Err => unreachable!(),
         }
     };
@@ -646,6 +667,9 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
         InlineAsmRegClass::Avr(_) => unimplemented!(),
         InlineAsmRegClass::Bpf(_) => unimplemented!(),
         InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
+        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
+        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
+        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
         InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
         InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
         InlineAsmRegClass::Msp430(_) => unimplemented!(),
@@ -692,21 +716,23 @@ impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
             && options.contains(InlineAsmOptions::ATT_SYNTAX);
 
         // Build the template string
-        let mut template_str = String::new();
+        let mut template_str = ".pushsection .text\n".to_owned();
+        if att_dialect {
+            template_str.push_str(".att_syntax\n");
+        }
         for piece in template {
             match *piece {
                 InlineAsmTemplatePiece::String(ref string) => {
-                    for line in string.lines() {
+                    let mut index = 0;
+                    while index < string.len() {
                         // NOTE: gcc does not allow inline comment, so remove them.
-                        let line =
-                            if let Some(index) = line.rfind("//") {
-                                &line[..index]
-                            }
-                            else {
-                                line
-                            };
-                        template_str.push_str(line);
-                        template_str.push('\n');
+                        let comment_index = string[index..].find("//")
+                            .map(|comment_index| comment_index + index)
+                            .unwrap_or(string.len());
+                        template_str.push_str(&string[index..comment_index]);
+                        index = string[comment_index..].find('\n')
+                            .map(|index| index + comment_index)
+                            .unwrap_or(string.len());
                     }
                 },
                 InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
@@ -719,6 +745,8 @@ impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                         }
 
                         GlobalAsmOperandRef::SymFn { instance } => {
+                            let function = get_fn(self, instance);
+                            self.add_used_function(function);
                             // TODO(@Amanieu): Additional mangling is needed on
                             // some targets to add a leading underscore (Mach-O)
                             // or byte count suffixes (x86 Windows).
@@ -727,6 +755,7 @@ impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                         }
 
                         GlobalAsmOperandRef::SymStatic { def_id } => {
+                            // TODO(antoyo): set the global variable as used.
                             // TODO(@Amanieu): Additional mangling is needed on
                             // some targets to add a leading underscore (Mach-O).
                             let instance = Instance::mono(self.tcx, def_id);
@@ -738,48 +767,51 @@ impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
             }
         }
 
-        let template_str =
-            if att_dialect {
-                format!(".att_syntax\n\t{}\n\t.intel_syntax noprefix", template_str)
-            }
-            else {
-                template_str
-            };
+        if att_dialect {
+            template_str.push_str("\n\t.intel_syntax noprefix");
+        }
         // NOTE: seems like gcc will put the asm in the wrong section, so set it to .text manually.
-        let template_str = format!(".pushsection .text\n{}\n.popsection", template_str);
+        template_str.push_str("\n.popsection");
         self.context.add_top_level_asm(None, &template_str);
     }
 }
 
 fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option<char>) -> Option<char> {
+    // The modifiers can be retrieved from
+    // https://gcc.gnu.org/onlinedocs/gcc/Modifiers.html#Modifiers
     match reg {
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => modifier,
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
         | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
-            unimplemented!()
+            if modifier == Some('v') { None } else { modifier }
+        }
+        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
+            unreachable!("clobber-only")
         }
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)  => unimplemented!(),
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None,
         InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => unimplemented!(),
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None,
         InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
         | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => unimplemented!(),
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
         InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
         | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
         | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
-            unimplemented!()
+            if modifier.is_none() {
+                Some('q')
+            } else {
+                modifier
+            }
         }
-        InlineAsmRegClass::Avr(_) => unimplemented!(),
-        InlineAsmRegClass::Bpf(_) => unimplemented!(),
-        InlineAsmRegClass::Hexagon(_) => unimplemented!(),
-        InlineAsmRegClass::Mips(_) => unimplemented!(),
-        InlineAsmRegClass::Msp430(_) => unimplemented!(),
-        InlineAsmRegClass::Nvptx(_) => unimplemented!(),
-        InlineAsmRegClass::PowerPC(_) => unimplemented!(),
+        InlineAsmRegClass::Hexagon(_) => None,
+        InlineAsmRegClass::Mips(_) => None,
+        InlineAsmRegClass::Nvptx(_) => None,
+        InlineAsmRegClass::PowerPC(_) => None,
         InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg)
-        | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => unimplemented!(),
-        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => unimplemented!(),
+        | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None,
+        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
+            unreachable!("clobber-only")
+        }
         InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
         | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
             None => if arch == InlineAsmArch::X86_64 { Some('q') } else { Some('k') },
@@ -803,16 +835,30 @@ fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option
             _ => unreachable!(),
         },
         InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None,
-        InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg0) => None,
-        InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::tmm_reg) => {
+        InlineAsmRegClass::X86(
+            X86InlineAsmRegClass::x87_reg
+            | X86InlineAsmRegClass::mmx_reg
+            | X86InlineAsmRegClass::kreg0
+            | X86InlineAsmRegClass::tmm_reg,
+        ) => {
             unreachable!("clobber-only")
         }
-        InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => unimplemented!(),
+        InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None,
+        InlineAsmRegClass::Bpf(_) => None,
+        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair)
+        | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw)
+        | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
+            Some('h') => Some('B'),
+            Some('l') => Some('A'),
+            _ => None,
+        },
+        InlineAsmRegClass::Avr(_) => None,
+        InlineAsmRegClass::S390x(_) => None,
+        InlineAsmRegClass::Msp430(_) => None,
+        InlineAsmRegClass::M68k(_) => None,
         InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
             bug!("LLVM backend does not support SPIR-V")
-        },
-        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => unimplemented!(),
-        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => unimplemented!(),
+        }
         InlineAsmRegClass::Err => unreachable!(),
     }
 }
diff --git a/compiler/rustc_codegen_gcc/src/attributes.rs b/compiler/rustc_codegen_gcc/src/attributes.rs
new file mode 100644
index 000000000..db841b1b5
--- /dev/null
+++ b/compiler/rustc_codegen_gcc/src/attributes.rs
@@ -0,0 +1,113 @@
+#[cfg(feature="master")]
+use gccjit::FnAttribute;
+use gccjit::Function;
+use rustc_attr::InstructionSetAttr;
+use rustc_codegen_ssa::target_features::tied_target_features;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_middle::ty;
+use rustc_session::Session;
+use rustc_span::symbol::sym;
+use smallvec::{smallvec, SmallVec};
+
+use crate::{context::CodegenCx, errors::TiedTargetFeatures};
+
+// Given a map from target_features to whether they are enabled or disabled,
+// ensure only valid combinations are allowed.
+pub fn check_tied_features(sess: &Session, features: &FxHashMap<&str, bool>) -> Option<&'static [&'static str]> {
+    for tied in tied_target_features(sess) {
+        // Tied features must be set to the same value, or not set at all
+        let mut tied_iter = tied.iter();
+        let enabled = features.get(tied_iter.next().unwrap());
+        if tied_iter.any(|feature| enabled != features.get(feature)) {
+            return Some(tied);
+        }
+    }
+    None
+}
+
+// TODO(antoyo): maybe move to a new module gcc_util.
+// To find a list of GCC's names, check https://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html
+fn to_gcc_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> {
+    let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch };
+    match (arch, s) {
+        ("x86", "sse4.2") => smallvec!["sse4.2", "crc32"],
+        ("x86", "pclmulqdq") => smallvec!["pclmul"],
+        ("x86", "rdrand") => smallvec!["rdrnd"],
+        ("x86", "bmi1") => smallvec!["bmi"],
+        ("x86", "cmpxchg16b") => smallvec!["cx16"],
+        ("x86", "avx512vaes") => smallvec!["vaes"],
+        ("x86", "avx512gfni") => smallvec!["gfni"],
+        ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"],
+        // NOTE: seems like GCC requires 'avx512bw' for 'avx512vbmi2'.
+        ("x86", "avx512vbmi2") => smallvec!["avx512vbmi2", "avx512bw"],
+        // NOTE: seems like GCC requires 'avx512bw' for 'avx512bitalg'.
+        ("x86", "avx512bitalg") => smallvec!["avx512bitalg", "avx512bw"],
+        ("aarch64", "rcpc2") => smallvec!["rcpc-immo"],
+        ("aarch64", "dpb") => smallvec!["ccpp"],
+        ("aarch64", "dpb2") => smallvec!["ccdp"],
+        ("aarch64", "frintts") => smallvec!["fptoint"],
+        ("aarch64", "fcma") => smallvec!["complxnum"],
+        ("aarch64", "pmuv3") => smallvec!["perfmon"],
+        ("aarch64", "paca") => smallvec!["pauth"],
+        ("aarch64", "pacg") => smallvec!["pauth"],
+        // Rust ties fp and neon together. In LLVM neon implicitly enables fp,
+        // but we manually enable neon when a feature only implicitly enables fp
+        ("aarch64", "f32mm") => smallvec!["f32mm", "neon"],
+        ("aarch64", "f64mm") => smallvec!["f64mm", "neon"],
+        ("aarch64", "fhm") => smallvec!["fp16fml", "neon"],
+        ("aarch64", "fp16") => smallvec!["fullfp16", "neon"],
+        ("aarch64", "jsconv") => smallvec!["jsconv", "neon"],
+        ("aarch64", "sve") => smallvec!["sve", "neon"],
+        ("aarch64", "sve2") => smallvec!["sve2", "neon"],
+        ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"],
+        ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"],
+        ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"],
+        ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"],
+        (_, s) => smallvec![s],
+    }
+}
+
+/// Composite function which sets GCC attributes for function depending on its AST (`#[attribute]`)
+/// attributes.
+pub fn from_fn_attrs<'gcc, 'tcx>(
+    cx: &CodegenCx<'gcc, 'tcx>,
+    #[cfg_attr(not(feature="master"), allow(unused_variables))]
+    func: Function<'gcc>,
+    instance: ty::Instance<'tcx>,
+) {
+    let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id());
+
+    let function_features =
+        codegen_fn_attrs.target_features.iter().map(|features| features.as_str()).collect::<Vec<&str>>();
+
+    if let Some(features) = check_tied_features(cx.tcx.sess, &function_features.iter().map(|features| (*features, true)).collect()) {
+        let span = cx.tcx
+            .get_attr(instance.def_id(), sym::target_feature)
+            .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span);
+        cx.tcx.sess.create_err(TiedTargetFeatures {
+            features: features.join(", "),
+            span,
+        })
+            .emit();
+        return;
+    }
+
+    let mut function_features = function_features
+        .iter()
+        .flat_map(|feat| to_gcc_features(cx.tcx.sess, feat).into_iter())
+        .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x {
+            InstructionSetAttr::ArmA32 => "-thumb-mode", // TODO(antoyo): support removing feature.
+            InstructionSetAttr::ArmT32 => "thumb-mode",
+        }))
+        .collect::<Vec<_>>();
+
+    // TODO(antoyo): check if we really need global backend features. (Maybe they could be applied
+    // globally?)
+    let mut global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str());
+    function_features.extend(&mut global_features);
+    let target_features = function_features.join(",");
+    if !target_features.is_empty() {
+        #[cfg(feature="master")]
+        func.add_attribute(FnAttribute::Target(&target_features));
+    }
+}
diff --git a/compiler/rustc_codegen_gcc/src/back/write.rs b/compiler/rustc_codegen_gcc/src/back/write.rs
index efcf18d31..5f54ac4eb 100644
--- a/compiler/rustc_codegen_gcc/src/back/write.rs
+++ b/compiler/rustc_codegen_gcc/src/back/write.rs
@@ -57,6 +57,7 @@ pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, _diag_han
                 if env::var("CG_GCCJIT_DUMP_TO_FILE").as_deref() == Ok("1") {
                     let _ = fs::create_dir("/tmp/gccjit_dumps");
                     let path = &format!("/tmp/gccjit_dumps/{}.c", module.name);
+                    context.set_debug_info(true);
                     context.dump_to_file(path, true);
                 }
                 context.compile_to_file(OutputKind::ObjectFile, obj_out.to_str().expect("path to str"));
diff --git a/compiler/rustc_codegen_gcc/src/base.rs b/compiler/rustc_codegen_gcc/src/base.rs
index d464bd3d1..dcd560b3d 100644
--- a/compiler/rustc_codegen_gcc/src/base.rs
+++ b/compiler/rustc_codegen_gcc/src/base.rs
@@ -8,6 +8,8 @@ use gccjit::{
 };
 use rustc_middle::dep_graph;
 use rustc_middle::ty::TyCtxt;
+#[cfg(feature="master")]
+use rustc_middle::mir::mono::Visibility;
 use rustc_middle::mir::mono::Linkage;
 use rustc_codegen_ssa::{ModuleCodegen, ModuleKind};
 use rustc_codegen_ssa::base::maybe_create_entry_wrapper;
@@ -20,6 +22,15 @@ use crate::GccContext;
 use crate::builder::Builder;
 use crate::context::CodegenCx;
 
+#[cfg(feature="master")]
+pub fn visibility_to_gcc(linkage: Visibility) -> gccjit::Visibility {
+    match linkage {
+        Visibility::Default => gccjit::Visibility::Default,
+        Visibility::Hidden => gccjit::Visibility::Hidden,
+        Visibility::Protected => gccjit::Visibility::Protected,
+    }
+}
+
 pub fn global_linkage_to_gcc(linkage: Linkage) -> GlobalKind {
     match linkage {
         Linkage::External => GlobalKind::Imported,
@@ -76,16 +87,34 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_i
         // Instantiate monomorphizations without filling out definitions yet...
         //let llvm_module = ModuleLlvm::new(tcx, &cgu_name.as_str());
         let context = Context::default();
+
+        context.add_command_line_option("-fexceptions");
+        context.add_driver_option("-fexceptions");
+
         // TODO(antoyo): only set on x86 platforms.
         context.add_command_line_option("-masm=intel");
         // TODO(antoyo): only add the following cli argument if the feature is supported.
         context.add_command_line_option("-msse2");
         context.add_command_line_option("-mavx2");
-        context.add_command_line_option("-msha");
-        context.add_command_line_option("-mpclmul");
         // FIXME(antoyo): the following causes an illegal instruction on vmovdqu64 in std_example on my CPU.
         // Only add if the CPU supports it.
-        //context.add_command_line_option("-mavx512f");
+        context.add_command_line_option("-msha");
+        context.add_command_line_option("-mpclmul");
+        context.add_command_line_option("-mfma");
+        context.add_command_line_option("-mfma4");
+        context.add_command_line_option("-m64");
+        context.add_command_line_option("-mbmi");
+        context.add_command_line_option("-mgfni");
+        //context.add_command_line_option("-mavxvnni"); // The CI doesn't support this option.
+        context.add_command_line_option("-mf16c");
+        context.add_command_line_option("-maes");
+        context.add_command_line_option("-mxsavec");
+        context.add_command_line_option("-mbmi2");
+        context.add_command_line_option("-mrtm");
+        context.add_command_line_option("-mvaes");
+        context.add_command_line_option("-mvpclmulqdq");
+        context.add_command_line_option("-mavx");
+
         for arg in &tcx.sess.opts.cg.llvm_args {
             context.add_command_line_option(arg);
         }
@@ -95,12 +124,20 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_i
         context.add_command_line_option("-fno-semantic-interposition");
         // NOTE: Rust relies on LLVM not doing TBAA (https://github.com/rust-lang/unsafe-code-guidelines/issues/292).
         context.add_command_line_option("-fno-strict-aliasing");
+        // NOTE: Rust relies on LLVM doing wrapping on overflow.
+        context.add_command_line_option("-fwrapv");
 
         if tcx.sess.opts.unstable_opts.function_sections.unwrap_or(tcx.sess.target.function_sections) {
             context.add_command_line_option("-ffunction-sections");
             context.add_command_line_option("-fdata-sections");
         }
 
+        if env::var("CG_GCCJIT_DUMP_RTL").as_deref() == Ok("1") {
+            context.add_command_line_option("-fdump-rtl-vregs");
+        }
+        if env::var("CG_GCCJIT_DUMP_TREE_ALL").as_deref() == Ok("1") {
+            context.add_command_line_option("-fdump-tree-all");
+        }
         if env::var("CG_GCCJIT_DUMP_CODE").as_deref() == Ok("1") {
             context.set_dump_code_on_compile(true);
         }
@@ -115,7 +152,7 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_i
             context.set_keep_intermediates(true);
         }
 
-        // TODO(bjorn3): Remove once unwinding is properly implemented
+        // NOTE: The codegen generates unrechable blocks.
         context.set_allow_unreachable_blocks(true);
 
         {
diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs
index e88c12716..a3c8142be 100644
--- a/compiler/rustc_codegen_gcc/src/builder.rs
+++ b/compiler/rustc_codegen_gcc/src/builder.rs
@@ -217,7 +217,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
 
                 let actual_ty = actual_val.get_type();
                 if expected_ty != actual_ty {
-                    if !actual_ty.is_vector() && !expected_ty.is_vector() && actual_ty.is_integral() && expected_ty.is_integral() && actual_ty.get_size() != expected_ty.get_size() {
+                    if !actual_ty.is_vector() && !expected_ty.is_vector() && (actual_ty.is_integral() && expected_ty.is_integral()) || (actual_ty.get_pointee().is_some() && expected_ty.get_pointee().is_some()) {
                         self.context.new_cast(None, actual_val, expected_ty)
                     }
                     else if on_stack_param_indices.contains(&index) {
@@ -226,6 +226,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                     else {
                         assert!(!((actual_ty.is_vector() && !expected_ty.is_vector()) || (!actual_ty.is_vector() && expected_ty.is_vector())), "{:?} ({}) -> {:?} ({}), index: {:?}[{}]", actual_ty, actual_ty.is_vector(), expected_ty, expected_ty.is_vector(), func_ptr, index);
                         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
+                        // TODO: remove bitcast now that vector types can be compared?
                         self.bitcast(actual_val, expected_ty)
                     }
                 }
@@ -279,21 +280,30 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     }
 
     fn function_ptr_call(&mut self, func_ptr: RValue<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>) -> RValue<'gcc> {
-        let args = self.check_ptr_call("call", func_ptr, args);
+        let gcc_func = func_ptr.get_type().dyncast_function_ptr_type().expect("function ptr");
+        let func_name = format!("{:?}", func_ptr);
+        let previous_arg_count = args.len();
+        let orig_args = args;
+        let args = {
+            let function_address_names = self.function_address_names.borrow();
+            let original_function_name = function_address_names.get(&func_ptr);
+            llvm::adjust_intrinsic_arguments(&self, gcc_func, args.into(), &func_name, original_function_name)
+        };
+        let args_adjusted = args.len() != previous_arg_count;
+        let args = self.check_ptr_call("call", func_ptr, &*args);
 
         // gccjit requires to use the result of functions, even when it's not used.
         // That's why we assign the result to a local or call add_eval().
-        let gcc_func = func_ptr.get_type().dyncast_function_ptr_type().expect("function ptr");
         let return_type = gcc_func.get_return_type();
         let void_type = self.context.new_type::<()>();
         let current_func = self.block.get_function();
 
         if return_type != void_type {
             unsafe { RETURN_VALUE_COUNT += 1 };
-            let result = current_func.new_local(None, return_type, &format!("ptrReturnValue{}", unsafe { RETURN_VALUE_COUNT }));
-            let func_name = format!("{:?}", func_ptr);
-            let args = llvm::adjust_intrinsic_arguments(&self, gcc_func, args, &func_name);
-            self.block.add_assignment(None, result, self.cx.context.new_call_through_ptr(None, func_ptr, &args));
+            let return_value = self.cx.context.new_call_through_ptr(None, func_ptr, &args);
+            let return_value = llvm::adjust_intrinsic_return_value(&self, return_value, &func_name, &args, args_adjusted, orig_args);
+            let result = current_func.new_local(None, return_value.get_type(), &format!("ptrReturnValue{}", unsafe { RETURN_VALUE_COUNT }));
+            self.block.add_assignment(None, result, return_value);
             result.to_rvalue()
         }
         else {
@@ -366,10 +376,10 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
     }
 }
 
-impl<'gcc, 'tcx> Deref for Builder<'_, 'gcc, 'tcx> {
+impl<'a, 'gcc, 'tcx> Deref for Builder<'a, 'gcc, 'tcx> {
     type Target = CodegenCx<'gcc, 'tcx>;
 
-    fn deref(&self) -> &Self::Target {
+    fn deref<'b>(&'b self) -> &'a Self::Target {
         self.cx
     }
 }
@@ -387,7 +397,7 @@ impl<'gcc, 'tcx> BackendTypes for Builder<'_, 'gcc, 'tcx> {
 }
 
 impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
-    fn build(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Self {
+    fn build(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Builder<'a, 'gcc, 'tcx> {
         Builder::with_cx(cx, block)
     }
 
@@ -444,17 +454,36 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         self.block.end_with_switch(None, value, default_block, &gcc_cases);
     }
 
-    fn invoke(
-        &mut self,
-        typ: Type<'gcc>,
-        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
-        func: RValue<'gcc>,
-        args: &[RValue<'gcc>],
-        then: Block<'gcc>,
-        catch: Block<'gcc>,
-        _funclet: Option<&Funclet>,
-    ) -> RValue<'gcc> {
-        // TODO(bjorn3): Properly implement unwinding.
+    #[cfg(feature="master")]
+    fn invoke(&mut self, typ: Type<'gcc>, _fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, _funclet: Option<&Funclet>) -> RValue<'gcc> {
+        let try_block = self.current_func().new_block("try");
+
+        let current_block = self.block.clone();
+        self.block = try_block;
+        let call = self.call(typ, None, func, args, None); // TODO(antoyo): use funclet here?
+        self.block = current_block;
+
+        let return_value = self.current_func()
+            .new_local(None, call.get_type(), "invokeResult");
+
+        try_block.add_assignment(None, return_value, call);
+
+        try_block.end_with_jump(None, then);
+
+        if self.cleanup_blocks.borrow().contains(&catch) {
+            self.block.add_try_finally(None, try_block, catch);
+        }
+        else {
+            self.block.add_try_catch(None, try_block, catch);
+        }
+
+        self.block.end_with_jump(None, then);
+
+        return_value.to_rvalue()
+    }
+
+    #[cfg(not(feature="master"))]
+    fn invoke(&mut self, typ: Type<'gcc>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, _funclet: Option<&Funclet>) -> RValue<'gcc> {
         let call_site = self.call(typ, None, func, args, None);
         let condition = self.context.new_rvalue_from_int(self.bool_type, 1);
         self.llbb().end_with_conditional(None, condition, then, catch);
@@ -542,6 +571,31 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     fn frem(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
+        // TODO(antoyo): add check in libgccjit since using the binary operator % causes the following error:
+        // during RTL pass: expand
+        // libgccjit.so: error: in expmed_mode_index, at expmed.h:240
+        // 0x7f0101d58dc6 expmed_mode_index
+        //     ../../../gcc/gcc/expmed.h:240
+        // 0x7f0101d58e35 expmed_op_cost_ptr
+        //     ../../../gcc/gcc/expmed.h:262
+        // 0x7f0101d594a1 sdiv_cost_ptr
+        //     ../../../gcc/gcc/expmed.h:531
+        // 0x7f0101d594f3 sdiv_cost
+        //     ../../../gcc/gcc/expmed.h:549
+        // 0x7f0101d6af7e expand_divmod(int, tree_code, machine_mode, rtx_def*, rtx_def*, rtx_def*, int, optab_methods)
+        //     ../../../gcc/gcc/expmed.cc:4356
+        // 0x7f0101d94f9e expand_expr_divmod
+        //     ../../../gcc/gcc/expr.cc:8929
+        // 0x7f0101d97a26 expand_expr_real_2(separate_ops*, rtx_def*, machine_mode, expand_modifier)
+        //     ../../../gcc/gcc/expr.cc:9566
+        // 0x7f0101bef6ef expand_gimple_stmt_1
+        //     ../../../gcc/gcc/cfgexpand.cc:3967
+        // 0x7f0101bef910 expand_gimple_stmt
+        //     ../../../gcc/gcc/cfgexpand.cc:4028
+        // 0x7f0101bf6ee7 expand_gimple_basic_block
+        //     ../../../gcc/gcc/cfgexpand.cc:6069
+        // 0x7f0101bf9194 execute
+        //     ../../../gcc/gcc/cfgexpand.cc:6795
         if a.get_type().is_compatible_with(self.cx.float_type) {
             let fmodf = self.context.get_builtin_function("fmodf");
             // FIXME(antoyo): this seems to produce the wrong result.
@@ -616,24 +670,29 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         a * b
     }
 
-    fn fadd_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
-        unimplemented!();
+    fn fadd_fast(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
+        // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
+        lhs + rhs
     }
 
-    fn fsub_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
-        unimplemented!();
+    fn fsub_fast(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
+        // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
+        lhs - rhs
     }
 
-    fn fmul_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
-        unimplemented!();
+    fn fmul_fast(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
+        // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
+        lhs * rhs
     }
 
-    fn fdiv_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
-        unimplemented!();
+    fn fdiv_fast(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
+        // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
+        lhs / rhs
     }
 
-    fn frem_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
-        unimplemented!();
+    fn frem_fast(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
+        // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
+        self.frem(lhs, rhs)
     }
 
     fn checked_binop(&mut self, oop: OverflowOp, typ: Ty<'_>, lhs: Self::Value, rhs: Self::Value) -> (Self::Value, Self::Value) {
@@ -722,7 +781,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
             }
             else if place.layout.is_gcc_immediate() {
                 let load = self.load(
-                    place.layout.gcc_type(self, false),
+                    place.layout.gcc_type(self),
                     place.llval,
                     place.align,
                 );
@@ -733,7 +792,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
             }
             else if let abi::Abi::ScalarPair(ref a, ref b) = place.layout.abi {
                 let b_offset = a.size(self).align_to(b.align(self).abi);
-                let pair_type = place.layout.gcc_type(self, false);
+                let pair_type = place.layout.gcc_type(self);
 
                 let mut load = |i, scalar: &abi::Scalar, align| {
                     let llptr = self.struct_gep(pair_type, place.llval, i as u64);
@@ -833,26 +892,31 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     fn gep(&mut self, _typ: Type<'gcc>, ptr: RValue<'gcc>, indices: &[RValue<'gcc>]) -> RValue<'gcc> {
-        let mut result = ptr;
+        let ptr_type = ptr.get_type();
+        let mut pointee_type = ptr.get_type();
+        // NOTE: we cannot use array indexing here like in inbounds_gep because array indexing is
+        // always considered in bounds in GCC (TODO(antoyo): to be verified).
+        // So, we have to cast to a number.
+        let mut result = self.context.new_bitcast(None, ptr, self.sizet_type);
+        // FIXME(antoyo): if there were more than 1 index, this code is probably wrong and would
+        // require dereferencing the pointer.
         for index in indices {
-            result = self.context.new_array_access(None, result, *index).get_address(None).to_rvalue();
+            pointee_type = pointee_type.get_pointee().expect("pointee type");
+            let pointee_size = self.context.new_rvalue_from_int(index.get_type(), pointee_type.get_size() as i32);
+            result = result + self.gcc_int_cast(*index * pointee_size, self.sizet_type);
         }
-        result
+        self.context.new_bitcast(None, result, ptr_type)
     }
 
     fn inbounds_gep(&mut self, _typ: Type<'gcc>, ptr: RValue<'gcc>, indices: &[RValue<'gcc>]) -> RValue<'gcc> {
-        // FIXME(antoyo): would be safer if doing the same thing (loop) as gep.
-        // TODO(antoyo): specify inbounds somehow.
-        match indices.len() {
-            1 => {
-                self.context.new_array_access(None, ptr, indices[0]).get_address(None)
-            },
-            2 => {
-                let array = ptr.dereference(None); // TODO(antoyo): assert that first index is 0?
-                self.context.new_array_access(None, array, indices[1]).get_address(None)
-            },
-            _ => unimplemented!(),
+        // NOTE: array indexing is always considered in bounds in GCC (TODO(antoyo): to be verified).
+        let mut indices = indices.into_iter();
+        let index = indices.next().expect("first index in inbounds_gep");
+        let mut result = self.context.new_array_access(None, ptr, *index);
+        for index in indices {
+            result = self.context.new_array_access(None, result, *index);
         }
+        result.get_address(None)
     }
 
     fn struct_gep(&mut self, value_type: Type<'gcc>, ptr: RValue<'gcc>, idx: u64) -> RValue<'gcc> {
@@ -1034,8 +1098,19 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         unimplemented!();
     }
 
-    fn extract_element(&mut self, _vec: RValue<'gcc>, _idx: RValue<'gcc>) -> RValue<'gcc> {
-        unimplemented!();
+    #[cfg(feature="master")]
+    fn extract_element(&mut self, vec: RValue<'gcc>, idx: RValue<'gcc>) -> RValue<'gcc> {
+        self.context.new_vector_access(None, vec, idx).to_rvalue()
+    }
+
+    #[cfg(not(feature="master"))]
+    fn extract_element(&mut self, vec: RValue<'gcc>, idx: RValue<'gcc>) -> RValue<'gcc> {
+        let vector_type = vec.get_type().unqualified().dyncast_vector().expect("Called extract_element on a non-vector type");
+        let element_type = vector_type.get_element_type();
+        let vec_num_units = vector_type.get_num_units();
+        let array_type = self.context.new_array_type(None, element_type, vec_num_units as u64);
+        let array = self.context.new_bitcast(None, vec, array_type).to_rvalue();
+        self.context.new_array_access(None, array, idx).to_rvalue()
     }
 
     fn vector_splat(&mut self, _num_elts: usize, _elt: RValue<'gcc>) -> RValue<'gcc> {
@@ -1116,22 +1191,52 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     fn set_personality_fn(&mut self, _personality: RValue<'gcc>) {
-        // TODO(antoyo)
+        #[cfg(feature="master")]
+        {
+            let personality = self.rvalue_as_function(_personality);
+            self.current_func().set_personality_function(personality);
+        }
+    }
+
+    #[cfg(feature="master")]
+    fn cleanup_landing_pad(&mut self, pers_fn: RValue<'gcc>) -> (RValue<'gcc>, RValue<'gcc>) {
+        self.set_personality_fn(pers_fn);
+
+        // NOTE: insert the current block in a variable so that a later call to invoke knows to
+        // generate a try/finally instead of a try/catch for this block.
+        self.cleanup_blocks.borrow_mut().insert(self.block);
+
+        let eh_pointer_builtin = self.cx.context.get_target_builtin_function("__builtin_eh_pointer");
+        let zero = self.cx.context.new_rvalue_zero(self.int_type);
+        let ptr = self.cx.context.new_call(None, eh_pointer_builtin, &[zero]);
+
+        let value1_type = self.u8_type.make_pointer();
+        let ptr = self.cx.context.new_cast(None, ptr, value1_type);
+        let value1 = ptr;
+        let value2 = zero; // TODO(antoyo): set the proper value here (the type of exception?).
+
+        (value1, value2)
     }
 
+    #[cfg(not(feature="master"))]
     fn cleanup_landing_pad(&mut self, _pers_fn: RValue<'gcc>) -> (RValue<'gcc>, RValue<'gcc>) {
-        (
-            self.current_func().new_local(None, self.u8_type.make_pointer(), "landing_pad0")
-                .to_rvalue(),
-            self.current_func().new_local(None, self.i32_type, "landing_pad1").to_rvalue(),
-        )
-        // TODO(antoyo): Properly implement unwinding.
-        // the above is just to make the compilation work as it seems
-        // rustc_codegen_ssa now calls the unwinding builder methods even on panic=abort.
+        let value1 = self.current_func().new_local(None, self.u8_type.make_pointer(), "landing_pad0")
+                .to_rvalue();
+        let value2 = self.current_func().new_local(None, self.i32_type, "landing_pad1").to_rvalue();
+        (value1, value2)
+    }
+
+    #[cfg(feature="master")]
+    fn resume(&mut self, exn0: RValue<'gcc>, _exn1: RValue<'gcc>) {
+        let exn_type = exn0.get_type();
+        let exn = self.context.new_cast(None, exn0, exn_type);
+        let unwind_resume = self.context.get_target_builtin_function("__builtin_unwind_resume");
+        self.llbb().add_eval(None, self.context.new_call(None, unwind_resume, &[exn]));
+        self.unreachable();
     }
 
+    #[cfg(not(feature="master"))]
     fn resume(&mut self, _exn0: RValue<'gcc>, _exn1: RValue<'gcc>) {
-        // TODO(bjorn3): Properly implement unwinding.
         self.unreachable();
     }
 
@@ -1160,6 +1265,15 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     fn atomic_cmpxchg(&mut self, dst: RValue<'gcc>, cmp: RValue<'gcc>, src: RValue<'gcc>, order: AtomicOrdering, failure_order: AtomicOrdering, weak: bool) -> RValue<'gcc> {
         let expected = self.current_func().new_local(None, cmp.get_type(), "expected");
         self.llbb().add_assignment(None, expected, cmp);
+        // NOTE: gcc doesn't support a failure memory model that is stronger than the success
+        // memory model.
+        let order =
+            if failure_order as i32 > order as i32 {
+                failure_order
+            }
+            else {
+                order
+            };
         let success = self.compare_exchange(dst, expected, src, order, failure_order, weak);
 
         let pair_type = self.cx.type_struct(&[src.get_type(), self.bool_type], false);
@@ -1469,7 +1583,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
 
     #[cfg(feature="master")]
     pub fn shuffle_vector(&mut self, v1: RValue<'gcc>, v2: RValue<'gcc>, mask: RValue<'gcc>) -> RValue<'gcc> {
-        let struct_type = mask.get_type().is_struct().expect("mask of struct type");
+        let struct_type = mask.get_type().is_struct().expect("mask should be of struct type");
 
         // TODO(antoyo): use a recursive unqualified() here.
         let vector_type = v1.get_type().unqualified().dyncast_vector().expect("vector type");
@@ -1501,22 +1615,17 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             vector_elements.push(self.context.new_rvalue_zero(mask_element_type));
         }
 
-        let array_type = self.context.new_array_type(None, element_type, vec_num_units as i32);
         let result_type = self.context.new_vector_type(element_type, mask_num_units as u64);
         let (v1, v2) =
             if vec_num_units < mask_num_units {
                 // NOTE: the mask needs to be the same length as the input vectors, so join the 2
                 // vectors and create a dummy second vector.
-                // TODO(antoyo): switch to using new_vector_access.
-                let array = self.context.new_bitcast(None, v1, array_type);
                 let mut elements = vec![];
                 for i in 0..vec_num_units {
-                    elements.push(self.context.new_array_access(None, array, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
+                    elements.push(self.context.new_vector_access(None, v1, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
                 }
-                // TODO(antoyo): switch to using new_vector_access.
-                let array = self.context.new_bitcast(None, v2, array_type);
                 for i in 0..(mask_num_units - vec_num_units) {
-                    elements.push(self.context.new_array_access(None, array, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
+                    elements.push(self.context.new_vector_access(None, v2, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
                 }
                 let v1 = self.context.new_rvalue_from_vector(None, result_type, &elements);
                 let zero = self.context.new_rvalue_zero(element_type);
@@ -1536,10 +1645,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             // NOTE: if padding was added, only select the number of elements of the masks to
             // remove that padding in the result.
             let mut elements = vec![];
-            // TODO(antoyo): switch to using new_vector_access.
-            let array = self.context.new_bitcast(None, result, array_type);
             for i in 0..mask_num_units {
-                elements.push(self.context.new_array_access(None, array, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
+                elements.push(self.context.new_vector_access(None, result, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
             }
             self.context.new_rvalue_from_vector(None, result_type, &elements)
         }
@@ -1558,18 +1665,20 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     where F: Fn(RValue<'gcc>, RValue<'gcc>, &'gcc Context<'gcc>) -> RValue<'gcc>
     {
         let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
+        let element_type = vector_type.get_element_type();
+        let mask_element_type = self.type_ix(element_type.get_size() as u64 * 8);
         let element_count = vector_type.get_num_units();
         let mut vector_elements = vec![];
         for i in 0..element_count {
             vector_elements.push(i);
         }
-        let mask_type = self.context.new_vector_type(self.int_type, element_count as u64);
+        let mask_type = self.context.new_vector_type(mask_element_type, element_count as u64);
         let mut shift = 1;
         let mut res = src;
         while shift < element_count {
             let vector_elements: Vec<_> =
                 vector_elements.iter()
-                    .map(|i| self.context.new_rvalue_from_int(self.int_type, ((i + shift) % element_count) as i32))
+                    .map(|i| self.context.new_rvalue_from_int(mask_element_type, ((i + shift) % element_count) as i32))
                     .collect();
             let mask = self.context.new_rvalue_from_vector(None, mask_type, &vector_elements);
             let shifted = self.context.new_rvalue_vector_perm(None, res, res, mask);
@@ -1581,7 +1690,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     }
 
     #[cfg(not(feature="master"))]
-    pub fn vector_reduce<F>(&mut self, src: RValue<'gcc>, op: F) -> RValue<'gcc>
+    pub fn vector_reduce<F>(&mut self, _src: RValue<'gcc>, _op: F) -> RValue<'gcc>
     where F: Fn(RValue<'gcc>, RValue<'gcc>, &'gcc Context<'gcc>) -> RValue<'gcc>
     {
         unimplemented!();
@@ -1595,15 +1704,47 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         unimplemented!();
     }
 
+    #[cfg(feature="master")]
+    pub fn vector_reduce_fadd(&mut self, acc: RValue<'gcc>, src: RValue<'gcc>) -> RValue<'gcc> {
+        let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
+        let element_count = vector_type.get_num_units();
+        (0..element_count).into_iter()
+            .map(|i| self.context
+                .new_vector_access(None, src, self.context.new_rvalue_from_int(self.int_type, i as _))
+                .to_rvalue())
+            .fold(acc, |x, i| x + i)
+    }
+
+    #[cfg(not(feature="master"))]
+    pub fn vector_reduce_fadd(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
+        unimplemented!();
+    }
+
     pub fn vector_reduce_fmul_fast(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
         unimplemented!();
     }
 
+    #[cfg(feature="master")]
+    pub fn vector_reduce_fmul(&mut self, acc: RValue<'gcc>, src: RValue<'gcc>) -> RValue<'gcc> {
+        let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
+        let element_count = vector_type.get_num_units();
+        (0..element_count).into_iter()
+            .map(|i| self.context
+                .new_vector_access(None, src, self.context.new_rvalue_from_int(self.int_type, i as _))
+                .to_rvalue())
+            .fold(acc, |x, i| x * i)
+    }
+
+    #[cfg(not(feature="master"))]
+    pub fn vector_reduce_fmul(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
+        unimplemented!()
+    }
+
     // Inspired by Hacker's Delight min implementation.
     pub fn vector_reduce_min(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
         self.vector_reduce(src, |a, b, context| {
             let differences_or_zeros = difference_or_zero(a, b, context);
-            context.new_binary_op(None, BinaryOp::Minus, a.get_type(), a, differences_or_zeros)
+            context.new_binary_op(None, BinaryOp::Plus, b.get_type(), b, differences_or_zeros)
         })
     }
 
@@ -1611,38 +1752,148 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     pub fn vector_reduce_max(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
         self.vector_reduce(src, |a, b, context| {
             let differences_or_zeros = difference_or_zero(a, b, context);
-            context.new_binary_op(None, BinaryOp::Plus, b.get_type(), b, differences_or_zeros)
+            context.new_binary_op(None, BinaryOp::Minus, a.get_type(), a, differences_or_zeros)
         })
     }
 
+    fn vector_extremum(&mut self, a: RValue<'gcc>, b: RValue<'gcc>, direction: ExtremumOperation) -> RValue<'gcc> {
+        let vector_type = a.get_type();
+
+        // mask out the NaNs in b and replace them with the corresponding lane in a, so when a and
+        // b get compared & spliced together, we get the numeric values instead of NaNs.
+        let b_nan_mask = self.context.new_comparison(None, ComparisonOp::NotEquals, b, b);
+        let mask_type = b_nan_mask.get_type();
+        let b_nan_mask_inverted = self.context.new_unary_op(None, UnaryOp::BitwiseNegate, mask_type, b_nan_mask);
+        let a_cast = self.context.new_bitcast(None, a, mask_type);
+        let b_cast = self.context.new_bitcast(None, b, mask_type);
+        let res = (b_nan_mask & a_cast) | (b_nan_mask_inverted & b_cast);
+        let b = self.context.new_bitcast(None, res, vector_type);
+
+        // now do the actual comparison
+        let comparison_op = match direction {
+            ExtremumOperation::Min => ComparisonOp::LessThan,
+            ExtremumOperation::Max => ComparisonOp::GreaterThan,
+        };
+        let cmp = self.context.new_comparison(None, comparison_op, a, b);
+        let cmp_inverted = self.context.new_unary_op(None, UnaryOp::BitwiseNegate, cmp.get_type(), cmp);
+        let res = (cmp & a_cast) | (cmp_inverted & res);
+        self.context.new_bitcast(None, res, vector_type)
+    }
+
+    pub fn vector_fmin(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
+        self.vector_extremum(a, b, ExtremumOperation::Min)
+    }
+
+    #[cfg(feature="master")]
+    pub fn vector_reduce_fmin(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
+        let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
+        let element_count = vector_type.get_num_units();
+        let mut acc = self.context.new_vector_access(None, src, self.context.new_rvalue_zero(self.int_type)).to_rvalue();
+        for i in 1..element_count {
+            let elem = self.context
+                .new_vector_access(None, src, self.context.new_rvalue_from_int(self.int_type, i as _))
+                .to_rvalue();
+            let cmp = self.context.new_comparison(None, ComparisonOp::LessThan, acc, elem);
+            acc = self.select(cmp, acc, elem);
+        }
+        acc
+    }
+
+    #[cfg(not(feature="master"))]
+    pub fn vector_reduce_fmin(&mut self, _src: RValue<'gcc>) -> RValue<'gcc> {
+        unimplemented!();
+    }
+
+    pub fn vector_fmax(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
+        self.vector_extremum(a, b, ExtremumOperation::Max)
+    }
+
+    #[cfg(feature="master")]
+    pub fn vector_reduce_fmax(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
+        let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
+        let element_count = vector_type.get_num_units();
+        let mut acc = self.context.new_vector_access(None, src, self.context.new_rvalue_zero(self.int_type)).to_rvalue();
+        for i in 1..element_count {
+            let elem = self.context
+                .new_vector_access(None, src, self.context.new_rvalue_from_int(self.int_type, i as _))
+                .to_rvalue();
+            let cmp = self.context.new_comparison(None, ComparisonOp::GreaterThan, acc, elem);
+            acc = self.select(cmp, acc, elem);
+        }
+        acc
+    }
+
+    #[cfg(not(feature="master"))]
+    pub fn vector_reduce_fmax(&mut self, _src: RValue<'gcc>) -> RValue<'gcc> {
+        unimplemented!();
+    }
+
     pub fn vector_select(&mut self, cond: RValue<'gcc>, then_val: RValue<'gcc>, else_val: RValue<'gcc>) -> RValue<'gcc> {
         // cond is a vector of integers, not of bools.
-        let cond_type = cond.get_type();
-        let vector_type = cond_type.unqualified().dyncast_vector().expect("vector type");
+        let vector_type = cond.get_type().unqualified().dyncast_vector().expect("vector type");
         let num_units = vector_type.get_num_units();
         let element_type = vector_type.get_element_type();
+
+        #[cfg(feature="master")]
+        let (cond, element_type) = {
+            let then_val_vector_type = then_val.get_type().dyncast_vector().expect("vector type");
+            let then_val_element_type = then_val_vector_type.get_element_type();
+            let then_val_element_size = then_val_element_type.get_size();
+
+            // NOTE: the mask needs to be of the same size as the other arguments in order for the &
+            // operation to work.
+            if then_val_element_size != element_type.get_size() {
+                let new_element_type = self.type_ix(then_val_element_size as u64 * 8);
+                let new_vector_type = self.context.new_vector_type(new_element_type, num_units as u64);
+                let cond = self.context.convert_vector(None, cond, new_vector_type);
+                (cond, new_element_type)
+            }
+            else {
+                (cond, element_type)
+            }
+        };
+
+        let cond_type = cond.get_type();
+
         let zeros = vec![self.context.new_rvalue_zero(element_type); num_units];
         let zeros = self.context.new_rvalue_from_vector(None, cond_type, &zeros);
 
+        let result_type = then_val.get_type();
+
         let masks = self.context.new_comparison(None, ComparisonOp::NotEquals, cond, zeros);
+        // NOTE: masks is a vector of integers, but the values can be vectors of floats, so use bitcast to make
+        // the & operation work.
+        let then_val = self.bitcast_if_needed(then_val, masks.get_type());
         let then_vals = masks & then_val;
 
-        let ones = vec![self.context.new_rvalue_one(element_type); num_units];
-        let ones = self.context.new_rvalue_from_vector(None, cond_type, &ones);
-        let inverted_masks = masks + ones;
+        let minus_ones = vec![self.context.new_rvalue_from_int(element_type, -1); num_units];
+        let minus_ones = self.context.new_rvalue_from_vector(None, cond_type, &minus_ones);
+        let inverted_masks = masks ^ minus_ones;
         // NOTE: sometimes, the type of else_val can be different than the type of then_val in
         // libgccjit (vector of int vs vector of int32_t), but they should be the same for the AND
         // operation to work.
+        // TODO: remove bitcast now that vector types can be compared?
         let else_val = self.context.new_bitcast(None, else_val, then_val.get_type());
         let else_vals = inverted_masks & else_val;
 
-        then_vals | else_vals
+        let res = then_vals | else_vals;
+        self.bitcast_if_needed(res, result_type)
     }
 }
 
 fn difference_or_zero<'gcc>(a: RValue<'gcc>, b: RValue<'gcc>, context: &'gcc Context<'gcc>) -> RValue<'gcc> {
     let difference = a - b;
     let masks = context.new_comparison(None, ComparisonOp::GreaterThanEquals, b, a);
+    // NOTE: masks is a vector of integers, but the values can be vectors of floats, so use bitcast to make
+    // the & operation work.
+    let a_type = a.get_type();
+    let masks =
+        if masks.get_type() != a_type {
+            context.new_bitcast(None, masks, a_type)
+        }
+        else {
+            masks
+        };
     difference & masks
 }
 
diff --git a/compiler/rustc_codegen_gcc/src/callee.rs b/compiler/rustc_codegen_gcc/src/callee.rs
index 9e3a22ee0..ba1e86562 100644
--- a/compiler/rustc_codegen_gcc/src/callee.rs
+++ b/compiler/rustc_codegen_gcc/src/callee.rs
@@ -1,9 +1,10 @@
-use gccjit::{FunctionType, RValue};
-use rustc_codegen_ssa::traits::BaseTypeMethods;
+#[cfg(feature="master")]
+use gccjit::{FnAttribute, Visibility};
+use gccjit::{FunctionType, Function};
 use rustc_middle::ty::{self, Instance, TypeVisitableExt};
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
 
-use crate::abi::FnAbiGccExt;
+use crate::attributes;
 use crate::context::CodegenCx;
 
 /// Codegens a reference to a fn/method item, monomorphizing and
@@ -13,22 +14,26 @@ use crate::context::CodegenCx;
 ///
 /// - `cx`: the crate context
 /// - `instance`: the instance to be instantiated
-pub fn get_fn<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, instance: Instance<'tcx>) -> RValue<'gcc> {
+pub fn get_fn<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, instance: Instance<'tcx>) -> Function<'gcc> {
     let tcx = cx.tcx();
 
     assert!(!instance.substs.needs_infer());
     assert!(!instance.substs.has_escaping_bound_vars());
 
+    let sym = tcx.symbol_name(instance).name;
+
     if let Some(&func) = cx.function_instances.borrow().get(&instance) {
         return func;
     }
 
-    let sym = tcx.symbol_name(instance).name;
-
     let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
 
     let func =
-        if let Some(func) = cx.get_declared_value(&sym) {
+        if let Some(_func) = cx.get_declared_value(&sym) {
+            // FIXME(antoyo): we never reach this because get_declared_value only returns global variables
+            // and here we try to get a function.
+            unreachable!();
+            /*
             // Create a fn pointer with the new signature.
             let ptrty = fn_abi.ptr_to_gcc_type(cx);
 
@@ -61,13 +66,105 @@ pub fn get_fn<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, instance: Instance<'tcx>)
             }
             else {
                 func
-            }
+            }*/
         }
         else {
             cx.linkage.set(FunctionType::Extern);
             let func = cx.declare_fn(&sym, &fn_abi);
 
+            attributes::from_fn_attrs(cx, func, instance);
+
+            let instance_def_id = instance.def_id();
+
             // TODO(antoyo): set linkage and attributes.
+
+            // Apply an appropriate linkage/visibility value to our item that we
+            // just declared.
+            //
+            // This is sort of subtle. Inside our codegen unit we started off
+            // compilation by predefining all our own `MonoItem` instances. That
+            // is, everything we're codegenning ourselves is already defined. That
+            // means that anything we're actually codegenning in this codegen unit
+            // will have hit the above branch in `get_declared_value`. As a result,
+            // we're guaranteed here that we're declaring a symbol that won't get
+            // defined, or in other words we're referencing a value from another
+            // codegen unit or even another crate.
+            //
+            // So because this is a foreign value we blanket apply an external
+            // linkage directive because it's coming from a different object file.
+            // The visibility here is where it gets tricky. This symbol could be
+            // referencing some foreign crate or foreign library (an `extern`
+            // block) in which case we want to leave the default visibility. We may
+            // also, though, have multiple codegen units. It could be a
+            // monomorphization, in which case its expected visibility depends on
+            // whether we are sharing generics or not. The important thing here is
+            // that the visibility we apply to the declaration is the same one that
+            // has been applied to the definition (wherever that definition may be).
+            let is_generic = instance.substs.non_erasable_generics().next().is_some();
+
+            if is_generic {
+                // This is a monomorphization. Its expected visibility depends
+                // on whether we are in share-generics mode.
+
+                if cx.tcx.sess.opts.share_generics() {
+                    // We are in share_generics mode.
+
+                    if let Some(instance_def_id) = instance_def_id.as_local() {
+                        // This is a definition from the current crate. If the
+                        // definition is unreachable for downstream crates or
+                        // the current crate does not re-export generics, the
+                        // definition of the instance will have been declared
+                        // as `hidden`.
+                        if cx.tcx.is_unreachable_local_definition(instance_def_id)
+                            || !cx.tcx.local_crate_exports_generics()
+                        {
+                            #[cfg(feature="master")]
+                            func.add_attribute(FnAttribute::Visibility(Visibility::Hidden));
+                        }
+                    } else {
+                        // This is a monomorphization of a generic function
+                        // defined in an upstream crate.
+                        if instance.upstream_monomorphization(tcx).is_some() {
+                            // This is instantiated in another crate. It cannot
+                            // be `hidden`.
+                        } else {
+                            // This is a local instantiation of an upstream definition.
+                            // If the current crate does not re-export it
+                            // (because it is a C library or an executable), it
+                            // will have been declared `hidden`.
+                            if !cx.tcx.local_crate_exports_generics() {
+                                #[cfg(feature="master")]
+                                func.add_attribute(FnAttribute::Visibility(Visibility::Hidden));
+                            }
+                        }
+                    }
+                } else {
+                    // When not sharing generics, all instances are in the same
+                    // crate and have hidden visibility
+                    #[cfg(feature="master")]
+                    func.add_attribute(FnAttribute::Visibility(Visibility::Hidden));
+                }
+            } else {
+                // This is a non-generic function
+                if cx.tcx.is_codegened_item(instance_def_id) {
+                    // This is a function that is instantiated in the local crate
+
+                    if instance_def_id.is_local() {
+                        // This is function that is defined in the local crate.
+                        // If it is not reachable, it is hidden.
+                        if !cx.tcx.is_reachable_non_generic(instance_def_id) {
+                            #[cfg(feature="master")]
+                            func.add_attribute(FnAttribute::Visibility(Visibility::Hidden));
+                        }
+                    } else {
+                        // This is a function from an upstream crate that has
+                        // been instantiated here. These are always hidden.
+                        #[cfg(feature="master")]
+                        func.add_attribute(FnAttribute::Visibility(Visibility::Hidden));
+                    }
+                }
+            }
+
             func
         };
 
diff --git a/compiler/rustc_codegen_gcc/src/common.rs b/compiler/rustc_codegen_gcc/src/common.rs
index c939da9ce..ac04b61a3 100644
--- a/compiler/rustc_codegen_gcc/src/common.rs
+++ b/compiler/rustc_codegen_gcc/src/common.rs
@@ -36,7 +36,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
 pub fn bytes_in_context<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, bytes: &[u8]) -> RValue<'gcc> {
     let context = &cx.context;
     let byte_type = context.new_type::<u8>();
-    let typ = context.new_array_type(None, byte_type, bytes.len() as i32);
+    let typ = context.new_array_type(None, byte_type, bytes.len() as u64);
     let elements: Vec<_> =
         bytes.iter()
         .map(|&byte| context.new_rvalue_from_int(byte_type, byte as i32))
@@ -73,6 +73,11 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         }
     }
 
+    fn const_poison(&self, typ: Type<'gcc>) -> RValue<'gcc> {
+        // No distinction between undef and poison.
+        self.const_undef(typ)
+    }
+
     fn const_int(&self, typ: Type<'gcc>, int: i64) -> RValue<'gcc> {
         self.gcc_int(typ, int)
     }
@@ -115,8 +120,8 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         self.const_uint(self.usize_type, i)
     }
 
-    fn const_u8(&self, _i: u8) -> RValue<'gcc> {
-        unimplemented!();
+    fn const_u8(&self, i: u8) -> RValue<'gcc> {
+        self.const_uint(self.type_u8(), i as u64)
     }
 
     fn const_real(&self, typ: Type<'gcc>, val: f64) -> RValue<'gcc> {
@@ -133,7 +138,7 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
             .1;
         let len = s.len();
         let cs = self.const_ptrcast(str_global.get_address(None),
-            self.type_ptr_to(self.layout_of(self.tcx.types.str_).gcc_type(self, true)),
+            self.type_ptr_to(self.layout_of(self.tcx.types.str_).gcc_type(self)),
         );
         (cs, self.const_usize(len as u64))
     }
@@ -174,8 +179,18 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                 }
 
                 let value = self.const_uint_big(self.type_ix(bitsize), data);
-                // TODO(bjorn3): assert size is correct
-                self.const_bitcast(value, ty)
+                let bytesize = layout.size(self).bytes();
+                if bitsize > 1 && ty.is_integral() && bytesize as u32 == ty.get_size() {
+                    // NOTE: since the intrinsic _xabort is called with a bitcast, which
+                    // is non-const, but expects a constant, do a normal cast instead of a bitcast.
+                    // FIXME(antoyo): fix bitcast to work in constant contexts.
+                    // TODO(antoyo): perhaps only use bitcast for pointers?
+                    self.context.new_cast(None, value, ty)
+                }
+                else {
+                    // TODO(bjorn3): assert size is correct
+                    self.const_bitcast(value, ty)
+                }
             }
             Scalar::Ptr(ptr, _size) => {
                 let (alloc_id, offset) = ptr.into_parts();
@@ -227,11 +242,11 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
 
     fn from_const_alloc(&self, layout: TyAndLayout<'tcx>, alloc: ConstAllocation<'tcx>, offset: Size) -> PlaceRef<'tcx, RValue<'gcc>> {
         assert_eq!(alloc.inner().align, layout.align.abi);
-        let ty = self.type_ptr_to(layout.gcc_type(self, true));
+        let ty = self.type_ptr_to(layout.gcc_type(self));
         let value =
             if layout.size == Size::ZERO {
                 let value = self.const_usize(alloc.inner().align.bytes());
-                self.context.new_cast(None, value, ty)
+                self.const_bitcast(value, ty)
             }
             else {
                 let init = const_alloc_to_gcc(self, alloc);
diff --git a/compiler/rustc_codegen_gcc/src/consts.rs b/compiler/rustc_codegen_gcc/src/consts.rs
index dc41cb761..792ab8f89 100644
--- a/compiler/rustc_codegen_gcc/src/consts.rs
+++ b/compiler/rustc_codegen_gcc/src/consts.rs
@@ -1,8 +1,8 @@
-use gccjit::{GlobalKind, LValue, RValue, ToRValue, Type};
+#[cfg(feature = "master")]
+use gccjit::FnAttribute;
+use gccjit::{Function, GlobalKind, LValue, RValue, ToRValue, Type};
 use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods, DerivedTypeMethods, StaticMethods};
-use rustc_hir as hir;
-use rustc_hir::Node;
-use rustc_middle::{bug, span_bug};
+use rustc_middle::span_bug;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
 use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::ty::{self, Instance, Ty};
@@ -13,6 +13,7 @@ use rustc_target::abi::{self, Align, HasDataLayout, Primitive, Size, WrappingRan
 
 use crate::base;
 use crate::context::CodegenCx;
+use crate::errors::InvalidMinimumAlignment;
 use crate::type_of::LayoutGccExt;
 
 impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
@@ -30,6 +31,21 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 }
 
+fn set_global_alignment<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, gv: LValue<'gcc>, mut align: Align) {
+    // The target may require greater alignment for globals than the type does.
+    // Note: GCC and Clang also allow `__attribute__((aligned))` on variables,
+    // which can force it to be smaller. Rust doesn't support this yet.
+    if let Some(min) = cx.sess().target.min_global_align {
+        match Align::from_bits(min) {
+            Ok(min) => align = align.max(min),
+            Err(err) => {
+                cx.sess().emit_err(InvalidMinimumAlignment { err });
+            }
+        }
+    }
+    gv.set_alignment(align.bytes() as i32);
+}
+
 impl<'gcc, 'tcx> StaticMethods for CodegenCx<'gcc, 'tcx> {
     fn static_addr_of(&self, cv: RValue<'gcc>, align: Align, kind: Option<&str>) -> RValue<'gcc> {
         // TODO(antoyo): implement a proper rvalue comparison in libgccjit instead of doing the
@@ -79,9 +95,9 @@ impl<'gcc, 'tcx> StaticMethods for CodegenCx<'gcc, 'tcx> {
 
         let instance = Instance::mono(self.tcx, def_id);
         let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
-        let gcc_type = self.layout_of(ty).gcc_type(self, true);
+        let gcc_type = self.layout_of(ty).gcc_type(self);
 
-        // TODO(antoyo): set alignment.
+        set_global_alignment(self, global, self.align_of(ty));
 
         let value = self.bitcast_if_needed(value, gcc_type);
         global.global_set_initializer_rvalue(value);
@@ -158,12 +174,19 @@ impl<'gcc, 'tcx> StaticMethods for CodegenCx<'gcc, 'tcx> {
         // TODO(antoyo)
     }
 
-    fn add_compiler_used_global(&self, _global: RValue<'gcc>) {
-        // TODO(antoyo)
+    fn add_compiler_used_global(&self, global: RValue<'gcc>) {
+        // NOTE: seems like GCC does not make the distinction between compiler.used and used.
+        self.add_used_global(global);
     }
 }
 
 impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
+    #[cfg_attr(not(feature="master"), allow(unused_variables))]
+    pub fn add_used_function(&self, function: Function<'gcc>) {
+        #[cfg(feature = "master")]
+        function.add_attribute(FnAttribute::Used);
+    }
+
     pub fn static_addr_of_mut(&self, cv: RValue<'gcc>, align: Align, kind: Option<&str>) -> RValue<'gcc> {
         let global =
             match kind {
@@ -208,82 +231,59 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
         let sym = self.tcx.symbol_name(instance).name;
 
         let global =
-            if let Some(def_id) = def_id.as_local() {
-                let id = self.tcx.hir().local_def_id_to_hir_id(def_id);
-                let llty = self.layout_of(ty).gcc_type(self, true);
-                // FIXME: refactor this to work without accessing the HIR
-                let global = match self.tcx.hir().get(id) {
-                    Node::Item(&hir::Item { span, kind: hir::ItemKind::Static(..), .. }) => {
-                        if let Some(global) = self.get_declared_value(&sym) {
-                            if self.val_ty(global) != self.type_ptr_to(llty) {
-                                span_bug!(span, "Conflicting types for static");
-                            }
-                        }
-
-                        let is_tls = fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL);
-                        let global = self.declare_global(
-                            &sym,
-                            llty,
-                            GlobalKind::Exported,
-                            is_tls,
-                            fn_attrs.link_section,
-                        );
-
-                        if !self.tcx.is_reachable_non_generic(def_id) {
-                            // TODO(antoyo): set visibility.
-                        }
-
-                        global
-                    }
-
-                    Node::ForeignItem(&hir::ForeignItem {
-                        span: _,
-                        kind: hir::ForeignItemKind::Static(..),
-                        ..
-                    }) => {
-                        let fn_attrs = self.tcx.codegen_fn_attrs(def_id);
-                        check_and_apply_linkage(&self, &fn_attrs, ty, sym)
-                    }
-
-                    item => bug!("get_static: expected static, found {:?}", item),
-                };
+            if def_id.is_local() && !self.tcx.is_foreign_item(def_id) {
+            let llty = self.layout_of(ty).gcc_type(self);
+            if let Some(global) = self.get_declared_value(sym) {
+                if self.val_ty(global) != self.type_ptr_to(llty) {
+                    span_bug!(self.tcx.def_span(def_id), "Conflicting types for static");
+                }
+            }
 
-                global
+            let is_tls = fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL);
+            let global = self.declare_global(
+                &sym,
+                llty,
+                GlobalKind::Exported,
+                is_tls,
+                fn_attrs.link_section,
+            );
+
+            if !self.tcx.is_reachable_non_generic(def_id) {
+                // TODO(antoyo): set visibility.
             }
-            else {
-                // FIXME(nagisa): perhaps the map of externs could be offloaded to llvm somehow?
-                //debug!("get_static: sym={} item_attr={:?}", sym, self.tcx.item_attrs(def_id));
-
-                let attrs = self.tcx.codegen_fn_attrs(def_id);
-                let global = check_and_apply_linkage(&self, &attrs, ty, sym);
-
-                let needs_dll_storage_attr = false; // TODO(antoyo)
-
-                // If this assertion triggers, there's something wrong with commandline
-                // argument validation.
-                debug_assert!(
-                    !(self.tcx.sess.opts.cg.linker_plugin_lto.enabled()
-                        && self.tcx.sess.target.options.is_like_msvc
-                        && self.tcx.sess.opts.cg.prefer_dynamic)
-                );
-
-                if needs_dll_storage_attr {
-                    // This item is external but not foreign, i.e., it originates from an external Rust
-                    // crate. Since we don't know whether this crate will be linked dynamically or
-                    // statically in the final application, we always mark such symbols as 'dllimport'.
-                    // If final linkage happens to be static, we rely on compiler-emitted __imp_ stubs
-                    // to make things work.
-                    //
-                    // However, in some scenarios we defer emission of statics to downstream
-                    // crates, so there are cases where a static with an upstream DefId
-                    // is actually present in the current crate. We can find out via the
-                    // is_codegened_item query.
-                    if !self.tcx.is_codegened_item(def_id) {
-                        unimplemented!();
-                    }
+
+            global
+        } else {
+            check_and_apply_linkage(&self, &fn_attrs, ty, sym)
+        };
+
+        if !def_id.is_local() {
+            let needs_dll_storage_attr = false; // TODO(antoyo)
+
+            // If this assertion triggers, there's something wrong with commandline
+            // argument validation.
+            debug_assert!(
+                !(self.tcx.sess.opts.cg.linker_plugin_lto.enabled()
+                    && self.tcx.sess.target.options.is_like_msvc
+                    && self.tcx.sess.opts.cg.prefer_dynamic)
+            );
+
+            if needs_dll_storage_attr {
+                // This item is external but not foreign, i.e., it originates from an external Rust
+                // crate. Since we don't know whether this crate will be linked dynamically or
+                // statically in the final application, we always mark such symbols as 'dllimport'.
+                // If final linkage happens to be static, we rely on compiler-emitted __imp_ stubs
+                // to make things work.
+                //
+                // However, in some scenarios we defer emission of statics to downstream
+                // crates, so there are cases where a static with an upstream DefId
+                // is actually present in the current crate. We can find out via the
+                // is_codegened_item query.
+                if !self.tcx.is_codegened_item(def_id) {
+                    unimplemented!();
                 }
-                global
-            };
+            }
+        }
 
         // TODO(antoyo): set dll storage class.
 
@@ -357,7 +357,7 @@ pub fn codegen_static_initializer<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, def_id
 
 fn check_and_apply_linkage<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, attrs: &CodegenFnAttrs, ty: Ty<'tcx>, sym: &str) -> LValue<'gcc> {
     let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL);
-    let llty = cx.layout_of(ty).gcc_type(cx, true);
+    let gcc_type = cx.layout_of(ty).gcc_type(cx);
     if let Some(linkage) = attrs.import_linkage {
         // Declare a symbol `foo` with the desired linkage.
         let global1 = cx.declare_global_with_linkage(&sym, cx.type_i8(), base::global_linkage_to_gcc(linkage));
@@ -370,9 +370,10 @@ fn check_and_apply_linkage<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, attrs: &Codeg
         // zero.
         let mut real_name = "_rust_extern_with_linkage_".to_string();
         real_name.push_str(&sym);
-        let global2 = cx.define_global(&real_name, llty, is_tls, attrs.link_section);
+        let global2 = cx.define_global(&real_name, gcc_type, is_tls, attrs.link_section);
         // TODO(antoyo): set linkage.
-        global2.global_set_initializer_rvalue(global1.get_address(None));
+        let value = cx.const_ptrcast(global1.get_address(None), gcc_type);
+        global2.global_set_initializer_rvalue(value);
         // TODO(antoyo): use global_set_initializer() when it will work.
         global2
     }
@@ -386,6 +387,6 @@ fn check_and_apply_linkage<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, attrs: &Codeg
         // don't do this then linker errors can be generated where the linker
         // complains that one object files has a thread local version of the
         // symbol and another one doesn't.
-        cx.declare_global(&sym, llty, GlobalKind::Imported, is_tls, attrs.link_section)
+        cx.declare_global(&sym, gcc_type, GlobalKind::Imported, is_tls, attrs.link_section)
     }
 }
diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs
index 457006319..661681bdb 100644
--- a/compiler/rustc_codegen_gcc/src/context.rs
+++ b/compiler/rustc_codegen_gcc/src/context.rs
@@ -1,9 +1,10 @@
 use std::cell::{Cell, RefCell};
 
-use gccjit::{Block, CType, Context, Function, FunctionPtrType, FunctionType, LValue, RValue, Struct, Type};
+use gccjit::{Block, CType, Context, Function, FunctionPtrType, FunctionType, LValue, RValue, Type};
 use rustc_codegen_ssa::base::wants_msvc_seh;
 use rustc_codegen_ssa::traits::{
     BackendTypes,
+    BaseTypeMethods,
     MiscMethods,
 };
 use rustc_data_structures::base_n;
@@ -11,7 +12,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_middle::span_bug;
 use rustc_middle::mir::mono::CodegenUnit;
 use rustc_middle::ty::{self, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt};
-use rustc_middle::ty::layout::{FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, HasTyCtxt, LayoutError, TyAndLayout, LayoutOfHelpers};
+use rustc_middle::ty::layout::{FnAbiError, FnAbiOf, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, HasTyCtxt, LayoutError, TyAndLayout, LayoutOfHelpers};
 use rustc_session::Session;
 use rustc_span::{Span, source_map::respan};
 use rustc_target::abi::{call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx};
@@ -33,6 +34,7 @@ pub struct CodegenCx<'gcc, 'tcx> {
     // TODO(bjorn3): Can this field be removed?
     pub current_func: RefCell<Option<Function<'gcc>>>,
     pub normal_function_addresses: RefCell<FxHashSet<RValue<'gcc>>>,
+    pub function_address_names: RefCell<FxHashMap<RValue<'gcc>, String>>,
 
     pub functions: RefCell<FxHashMap<String, Function<'gcc>>>,
     pub intrinsics: RefCell<FxHashMap<String, Function<'gcc>>>,
@@ -78,12 +80,10 @@ pub struct CodegenCx<'gcc, 'tcx> {
 
     pub struct_types: RefCell<FxHashMap<Vec<Type<'gcc>>, Type<'gcc>>>,
 
-    pub types_with_fields_to_set: RefCell<FxHashMap<Type<'gcc>, (Struct<'gcc>, TyAndLayout<'tcx>)>>,
-
     /// Cache instances of monomorphic and polymorphic items
     pub instances: RefCell<FxHashMap<Instance<'tcx>, LValue<'gcc>>>,
     /// Cache function instances of monomorphic and polymorphic items
-    pub function_instances: RefCell<FxHashMap<Instance<'tcx>, RValue<'gcc>>>,
+    pub function_instances: RefCell<FxHashMap<Instance<'tcx>, Function<'gcc>>>,
     /// Cache generated vtables
     pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), RValue<'gcc>>>,
 
@@ -110,6 +110,7 @@ pub struct CodegenCx<'gcc, 'tcx> {
     local_gen_sym_counter: Cell<usize>,
 
     eh_personality: Cell<Option<RValue<'gcc>>>,
+    pub rust_try_fn: Cell<Option<(Type<'gcc>, Function<'gcc>)>>,
 
     pub pointee_infos: RefCell<FxHashMap<(Ty<'tcx>, Size), Option<PointeeInfo>>>,
 
@@ -119,6 +120,8 @@ pub struct CodegenCx<'gcc, 'tcx> {
     /// they can be dereferenced later.
     /// FIXME(antoyo): fix the rustc API to avoid having this hack.
     pub structs_as_pointer: RefCell<FxHashSet<RValue<'gcc>>>,
+
+    pub cleanup_blocks: RefCell<FxHashSet<Block<'gcc>>>,
 }
 
 impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
@@ -194,6 +197,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             context,
             current_func: RefCell::new(None),
             normal_function_addresses: Default::default(),
+            function_address_names: Default::default(),
             functions: RefCell::new(functions),
             intrinsics: RefCell::new(FxHashMap::default()),
 
@@ -243,11 +247,12 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             types: Default::default(),
             tcx,
             struct_types: Default::default(),
-            types_with_fields_to_set: Default::default(),
             local_gen_sym_counter: Cell::new(0),
             eh_personality: Cell::new(None),
+            rust_try_fn: Cell::new(None),
             pointee_infos: Default::default(),
             structs_as_pointer: Default::default(),
+            cleanup_blocks: Default::default(),
         }
     }
 
@@ -327,8 +332,9 @@ impl<'gcc, 'tcx> MiscMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
 
     fn get_fn(&self, instance: Instance<'tcx>) -> RValue<'gcc> {
         let func = get_fn(self, instance);
-        *self.current_func.borrow_mut() = Some(self.rvalue_as_function(func));
-        func
+        *self.current_func.borrow_mut() = Some(func);
+        // FIXME(antoyo): this is a wrong cast. That requires changing the compiler API.
+        unsafe { std::mem::transmute(func) }
     }
 
     fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> {
@@ -339,8 +345,7 @@ impl<'gcc, 'tcx> MiscMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                 self.intrinsics.borrow()[func_name].clone()
             }
             else {
-                let func = get_fn(self, instance);
-                self.rvalue_as_function(func)
+                get_fn(self, instance)
             };
         let ptr = func.get_address(None);
 
@@ -348,6 +353,7 @@ impl<'gcc, 'tcx> MiscMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         // FIXME(antoyo): the rustc API seems to call get_fn_addr() when not needed (e.g. for FFI).
 
         self.normal_function_addresses.borrow_mut().insert(ptr);
+        self.function_address_names.borrow_mut().insert(ptr, func_name.to_string());
 
         ptr
     }
@@ -377,31 +383,40 @@ impl<'gcc, 'tcx> MiscMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
             return llpersonality;
         }
         let tcx = self.tcx;
-        let llfn = match tcx.lang_items().eh_personality() {
-            Some(def_id) if !wants_msvc_seh(self.sess()) => self.get_fn_addr(
-                ty::Instance::resolve(
-                    tcx,
-                    ty::ParamEnv::reveal_all(),
-                    def_id,
-                    ty::List::empty(),
-                )
-                .unwrap().unwrap(),
-            ),
-            _ => {
-                let _name = if wants_msvc_seh(self.sess()) {
-                    "__CxxFrameHandler3"
-                } else {
-                    "rust_eh_personality"
-                };
-                //let func = self.declare_func(name, self.type_i32(), &[], true);
-                // FIXME(antoyo): this hack should not be needed. That will probably be removed when
-                // unwinding support is added.
-                self.context.new_rvalue_from_int(self.int_type, 0)
-            }
-        };
+        let func =
+            match tcx.lang_items().eh_personality() {
+                Some(def_id) if !wants_msvc_seh(self.sess()) => {
+                    let instance =
+                        ty::Instance::resolve(
+                            tcx,
+                            ty::ParamEnv::reveal_all(),
+                            def_id,
+                            ty::List::empty(),
+                        )
+                        .unwrap().unwrap();
+
+                    let symbol_name = tcx.symbol_name(instance).name;
+                    let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
+                    self.linkage.set(FunctionType::Extern);
+                    let func = self.declare_fn(symbol_name, &fn_abi);
+                    let func: RValue<'gcc> = unsafe { std::mem::transmute(func) };
+                    func
+                },
+                _ => {
+                    let name =
+                        if wants_msvc_seh(self.sess()) {
+                            "__CxxFrameHandler3"
+                        }
+                        else {
+                            "rust_eh_personality"
+                        };
+                    let func = self.declare_func(name, self.type_i32(), &[], true);
+                    unsafe { std::mem::transmute(func) }
+                }
+            };
         // TODO(antoyo): apply target cpu attributes.
-        self.eh_personality.set(Some(llfn));
-        llfn
+        self.eh_personality.set(Some(func));
+        func
     }
 
     fn sess(&self) -> &Session {
diff --git a/compiler/rustc_codegen_gcc/src/declare.rs b/compiler/rustc_codegen_gcc/src/declare.rs
index eae77508c..4748e7e4b 100644
--- a/compiler/rustc_codegen_gcc/src/declare.rs
+++ b/compiler/rustc_codegen_gcc/src/declare.rs
@@ -38,12 +38,10 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
         global
     }
 
-    /*pub fn declare_func(&self, name: &str, return_type: Type<'gcc>, params: &[Type<'gcc>], variadic: bool) -> RValue<'gcc> {
-        self.linkage.set(FunctionType::Exported);
-        let func = declare_raw_fn(self, name, () /*llvm::CCallConv*/, return_type, params, variadic);
-        // FIXME(antoyo): this is a wrong cast. That requires changing the compiler API.
-        unsafe { std::mem::transmute(func) }
-    }*/
+    pub fn declare_func(&self, name: &str, return_type: Type<'gcc>, params: &[Type<'gcc>], variadic: bool) -> Function<'gcc> {
+        self.linkage.set(FunctionType::Extern);
+        declare_raw_fn(self, name, () /*llvm::CCallConv*/, return_type, params, variadic)
+    }
 
     pub fn declare_global(&self, name: &str, ty: Type<'gcc>, global_kind: GlobalKind, is_tls: bool, link_section: Option<Symbol>) -> LValue<'gcc> {
         let global = self.context.new_global(None, global_kind, ty, name);
@@ -79,12 +77,11 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
         unsafe { std::mem::transmute(func) }
     }
 
-    pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> RValue<'gcc> {
+    pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Function<'gcc> {
         let (return_type, params, variadic, on_stack_param_indices) = fn_abi.gcc_type(self);
         let func = declare_raw_fn(self, name, () /*fn_abi.llvm_cconv()*/, return_type, &params, variadic);
         self.on_stack_function_params.borrow_mut().insert(func, on_stack_param_indices);
-        // FIXME(antoyo): this is a wrong cast. That requires changing the compiler API.
-        unsafe { std::mem::transmute(func) }
+        func
     }
 
     pub fn define_global(&self, name: &str, ty: Type<'gcc>, is_tls: bool, link_section: Option<Symbol>) -> LValue<'gcc> {
diff --git a/compiler/rustc_codegen_gcc/src/errors.rs b/compiler/rustc_codegen_gcc/src/errors.rs
index d0ba7e247..9305bd1e0 100644
--- a/compiler/rustc_codegen_gcc/src/errors.rs
+++ b/compiler/rustc_codegen_gcc/src/errors.rs
@@ -221,3 +221,18 @@ pub(crate) struct UnwindingInlineAsm {
     #[primary_span]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_invalid_minimum_alignment)]
+pub(crate) struct InvalidMinimumAlignment {
+    pub err: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_tied_target_features)]
+#[help]
+pub(crate) struct TiedTargetFeatures {
+    #[primary_span]
+    pub span: Span,
+    pub features: String,
+}
diff --git a/compiler/rustc_codegen_gcc/src/int.rs b/compiler/rustc_codegen_gcc/src/int.rs
index 0c5dab004..0cf120479 100644
--- a/compiler/rustc_codegen_gcc/src/int.rs
+++ b/compiler/rustc_codegen_gcc/src/int.rs
@@ -389,18 +389,22 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                 };
             self.context.new_comparison(None, op, cmp, self.context.new_rvalue_from_int(self.int_type, limit))
         }
+        else if a_type.get_pointee().is_some() && b_type.get_pointee().is_some() {
+            // NOTE: gcc cannot compare pointers to different objects, but rustc does that, so cast them to usize.
+            lhs = self.context.new_bitcast(None, lhs, self.usize_type);
+            rhs = self.context.new_bitcast(None, rhs, self.usize_type);
+            self.context.new_comparison(None, op.to_gcc_comparison(), lhs, rhs)
+        }
         else {
-            let left_type = lhs.get_type();
-            let right_type = rhs.get_type();
-            if left_type != right_type {
+            if a_type != b_type {
                 // NOTE: because libgccjit cannot compare function pointers.
-                if left_type.dyncast_function_ptr_type().is_some() && right_type.dyncast_function_ptr_type().is_some() {
+                if a_type.dyncast_function_ptr_type().is_some() && b_type.dyncast_function_ptr_type().is_some() {
                     lhs = self.context.new_cast(None, lhs, self.usize_type.make_pointer());
                     rhs = self.context.new_cast(None, rhs, self.usize_type.make_pointer());
                 }
                 // NOTE: hack because we try to cast a vector type to the same vector type.
-                else if format!("{:?}", left_type) != format!("{:?}", right_type) {
-                    rhs = self.context.new_cast(None, rhs, left_type);
+                else if format!("{:?}", a_type) != format!("{:?}", b_type) {
+                    rhs = self.context.new_cast(None, rhs, a_type);
                 }
             }
             self.context.new_comparison(None, op.to_gcc_comparison(), lhs, rhs)
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs b/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs
index fb6c38fa0..8a4559355 100644
--- a/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs
@@ -34,6 +34,7 @@ match name {
     "llvm.aarch64.dmb" => "__builtin_arm_dmb",
     "llvm.aarch64.dsb" => "__builtin_arm_dsb",
     "llvm.aarch64.isb" => "__builtin_arm_isb",
+    "llvm.aarch64.prefetch" => "__builtin_arm_prefetch",
     "llvm.aarch64.sve.aesd" => "__builtin_sve_svaesd_u8",
     "llvm.aarch64.sve.aese" => "__builtin_sve_svaese_u8",
     "llvm.aarch64.sve.aesimc" => "__builtin_sve_svaesimc_u8",
@@ -58,13 +59,22 @@ match name {
     "llvm.amdgcn.cubema" => "__builtin_amdgcn_cubema",
     "llvm.amdgcn.cubesc" => "__builtin_amdgcn_cubesc",
     "llvm.amdgcn.cubetc" => "__builtin_amdgcn_cubetc",
+    "llvm.amdgcn.cvt.f32.bf8" => "__builtin_amdgcn_cvt_f32_bf8",
+    "llvm.amdgcn.cvt.f32.fp8" => "__builtin_amdgcn_cvt_f32_fp8",
+    "llvm.amdgcn.cvt.pk.bf8.f32" => "__builtin_amdgcn_cvt_pk_bf8_f32",
+    "llvm.amdgcn.cvt.pk.f32.bf8" => "__builtin_amdgcn_cvt_pk_f32_bf8",
+    "llvm.amdgcn.cvt.pk.f32.fp8" => "__builtin_amdgcn_cvt_pk_f32_fp8",
+    "llvm.amdgcn.cvt.pk.fp8.f32" => "__builtin_amdgcn_cvt_pk_fp8_f32",
     "llvm.amdgcn.cvt.pk.i16" => "__builtin_amdgcn_cvt_pk_i16",
     "llvm.amdgcn.cvt.pk.u16" => "__builtin_amdgcn_cvt_pk_u16",
     "llvm.amdgcn.cvt.pk.u8.f32" => "__builtin_amdgcn_cvt_pk_u8_f32",
     "llvm.amdgcn.cvt.pknorm.i16" => "__builtin_amdgcn_cvt_pknorm_i16",
     "llvm.amdgcn.cvt.pknorm.u16" => "__builtin_amdgcn_cvt_pknorm_u16",
     "llvm.amdgcn.cvt.pkrtz" => "__builtin_amdgcn_cvt_pkrtz",
+    "llvm.amdgcn.cvt.sr.bf8.f32" => "__builtin_amdgcn_cvt_sr_bf8_f32",
+    "llvm.amdgcn.cvt.sr.fp8.f32" => "__builtin_amdgcn_cvt_sr_fp8_f32",
     "llvm.amdgcn.dispatch.id" => "__builtin_amdgcn_dispatch_id",
+    "llvm.amdgcn.ds.add.gs.reg.rtn" => "__builtin_amdgcn_ds_add_gs_reg_rtn",
     "llvm.amdgcn.ds.bpermute" => "__builtin_amdgcn_ds_bpermute",
     "llvm.amdgcn.ds.fadd.v2bf16" => "__builtin_amdgcn_ds_atomic_fadd_v2bf16",
     "llvm.amdgcn.ds.gws.barrier" => "__builtin_amdgcn_ds_gws_barrier",
@@ -74,12 +84,16 @@ match name {
     "llvm.amdgcn.ds.gws.sema.release.all" => "__builtin_amdgcn_ds_gws_sema_release_all",
     "llvm.amdgcn.ds.gws.sema.v" => "__builtin_amdgcn_ds_gws_sema_v",
     "llvm.amdgcn.ds.permute" => "__builtin_amdgcn_ds_permute",
+    "llvm.amdgcn.ds.sub.gs.reg.rtn" => "__builtin_amdgcn_ds_sub_gs_reg_rtn",
     "llvm.amdgcn.ds.swizzle" => "__builtin_amdgcn_ds_swizzle",
     "llvm.amdgcn.endpgm" => "__builtin_amdgcn_endpgm",
     "llvm.amdgcn.fdot2" => "__builtin_amdgcn_fdot2",
-    "llvm.amdgcn.fmed3" => "__builtin_amdgcn_fmed3",
+    "llvm.amdgcn.fdot2.bf16.bf16" => "__builtin_amdgcn_fdot2_bf16_bf16",
+    "llvm.amdgcn.fdot2.f16.f16" => "__builtin_amdgcn_fdot2_f16_f16",
+    "llvm.amdgcn.fdot2.f32.bf16" => "__builtin_amdgcn_fdot2_f32_bf16",
     "llvm.amdgcn.fmul.legacy" => "__builtin_amdgcn_fmul_legacy",
     "llvm.amdgcn.groupstaticsize" => "__builtin_amdgcn_groupstaticsize",
+    "llvm.amdgcn.iglp.opt" => "__builtin_amdgcn_iglp_opt",
     "llvm.amdgcn.implicit.buffer.ptr" => "__builtin_amdgcn_implicit_buffer_ptr",
     "llvm.amdgcn.implicitarg.ptr" => "__builtin_amdgcn_implicitarg_ptr",
     "llvm.amdgcn.interp.mov" => "__builtin_amdgcn_interp_mov",
@@ -93,11 +107,51 @@ match name {
     "llvm.amdgcn.lerp" => "__builtin_amdgcn_lerp",
     "llvm.amdgcn.mbcnt.hi" => "__builtin_amdgcn_mbcnt_hi",
     "llvm.amdgcn.mbcnt.lo" => "__builtin_amdgcn_mbcnt_lo",
+    "llvm.amdgcn.mfma.f32.16x16x16bf16.1k" => "__builtin_amdgcn_mfma_f32_16x16x16bf16_1k",
+    "llvm.amdgcn.mfma.f32.16x16x16f16" => "__builtin_amdgcn_mfma_f32_16x16x16f16",
+    "llvm.amdgcn.mfma.f32.16x16x1f32" => "__builtin_amdgcn_mfma_f32_16x16x1f32",
+    "llvm.amdgcn.mfma.f32.16x16x2bf16" => "__builtin_amdgcn_mfma_f32_16x16x2bf16",
+    "llvm.amdgcn.mfma.f32.16x16x32.bf8.bf8" => "__builtin_amdgcn_mfma_f32_16x16x32_bf8_bf8",
+    "llvm.amdgcn.mfma.f32.16x16x32.bf8.fp8" => "__builtin_amdgcn_mfma_f32_16x16x32_bf8_fp8",
+    "llvm.amdgcn.mfma.f32.16x16x32.fp8.bf8" => "__builtin_amdgcn_mfma_f32_16x16x32_fp8_bf8",
+    "llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8" => "__builtin_amdgcn_mfma_f32_16x16x32_fp8_fp8",
+    "llvm.amdgcn.mfma.f32.16x16x4bf16.1k" => "__builtin_amdgcn_mfma_f32_16x16x4bf16_1k",
+    "llvm.amdgcn.mfma.f32.16x16x4f16" => "__builtin_amdgcn_mfma_f32_16x16x4f16",
+    "llvm.amdgcn.mfma.f32.16x16x4f32" => "__builtin_amdgcn_mfma_f32_16x16x4f32",
+    "llvm.amdgcn.mfma.f32.16x16x8.xf32" => "__builtin_amdgcn_mfma_f32_16x16x8_xf32",
+    "llvm.amdgcn.mfma.f32.16x16x8bf16" => "__builtin_amdgcn_mfma_f32_16x16x8bf16",
+    "llvm.amdgcn.mfma.f32.32x32x16.bf8.bf8" => "__builtin_amdgcn_mfma_f32_32x32x16_bf8_bf8",
+    "llvm.amdgcn.mfma.f32.32x32x16.bf8.fp8" => "__builtin_amdgcn_mfma_f32_32x32x16_bf8_fp8",
+    "llvm.amdgcn.mfma.f32.32x32x16.fp8.bf8" => "__builtin_amdgcn_mfma_f32_32x32x16_fp8_bf8",
+    "llvm.amdgcn.mfma.f32.32x32x16.fp8.fp8" => "__builtin_amdgcn_mfma_f32_32x32x16_fp8_fp8",
+    "llvm.amdgcn.mfma.f32.32x32x1f32" => "__builtin_amdgcn_mfma_f32_32x32x1f32",
+    "llvm.amdgcn.mfma.f32.32x32x2bf16" => "__builtin_amdgcn_mfma_f32_32x32x2bf16",
+    "llvm.amdgcn.mfma.f32.32x32x2f32" => "__builtin_amdgcn_mfma_f32_32x32x2f32",
+    "llvm.amdgcn.mfma.f32.32x32x4.xf32" => "__builtin_amdgcn_mfma_f32_32x32x4_xf32",
+    "llvm.amdgcn.mfma.f32.32x32x4bf16" => "__builtin_amdgcn_mfma_f32_32x32x4bf16",
+    "llvm.amdgcn.mfma.f32.32x32x4bf16.1k" => "__builtin_amdgcn_mfma_f32_32x32x4bf16_1k",
+    "llvm.amdgcn.mfma.f32.32x32x4f16" => "__builtin_amdgcn_mfma_f32_32x32x4f16",
+    "llvm.amdgcn.mfma.f32.32x32x8bf16.1k" => "__builtin_amdgcn_mfma_f32_32x32x8bf16_1k",
+    "llvm.amdgcn.mfma.f32.32x32x8f16" => "__builtin_amdgcn_mfma_f32_32x32x8f16",
+    "llvm.amdgcn.mfma.f32.4x4x1f32" => "__builtin_amdgcn_mfma_f32_4x4x1f32",
+    "llvm.amdgcn.mfma.f32.4x4x2bf16" => "__builtin_amdgcn_mfma_f32_4x4x2bf16",
+    "llvm.amdgcn.mfma.f32.4x4x4bf16.1k" => "__builtin_amdgcn_mfma_f32_4x4x4bf16_1k",
+    "llvm.amdgcn.mfma.f32.4x4x4f16" => "__builtin_amdgcn_mfma_f32_4x4x4f16",
+    "llvm.amdgcn.mfma.f64.16x16x4f64" => "__builtin_amdgcn_mfma_f64_16x16x4f64",
+    "llvm.amdgcn.mfma.f64.4x4x4f64" => "__builtin_amdgcn_mfma_f64_4x4x4f64",
+    "llvm.amdgcn.mfma.i32.16x16x16i8" => "__builtin_amdgcn_mfma_i32_16x16x16i8",
+    "llvm.amdgcn.mfma.i32.16x16x32.i8" => "__builtin_amdgcn_mfma_i32_16x16x32_i8",
+    "llvm.amdgcn.mfma.i32.16x16x4i8" => "__builtin_amdgcn_mfma_i32_16x16x4i8",
+    "llvm.amdgcn.mfma.i32.32x32x16.i8" => "__builtin_amdgcn_mfma_i32_32x32x16_i8",
+    "llvm.amdgcn.mfma.i32.32x32x4i8" => "__builtin_amdgcn_mfma_i32_32x32x4i8",
+    "llvm.amdgcn.mfma.i32.32x32x8i8" => "__builtin_amdgcn_mfma_i32_32x32x8i8",
+    "llvm.amdgcn.mfma.i32.4x4x4i8" => "__builtin_amdgcn_mfma_i32_4x4x4i8",
     "llvm.amdgcn.mqsad.pk.u16.u8" => "__builtin_amdgcn_mqsad_pk_u16_u8",
     "llvm.amdgcn.mqsad.u32.u8" => "__builtin_amdgcn_mqsad_u32_u8",
     "llvm.amdgcn.msad.u8" => "__builtin_amdgcn_msad_u8",
     "llvm.amdgcn.perm" => "__builtin_amdgcn_perm",
     "llvm.amdgcn.permlane16" => "__builtin_amdgcn_permlane16",
+    "llvm.amdgcn.permlane64" => "__builtin_amdgcn_permlane64",
     "llvm.amdgcn.permlanex16" => "__builtin_amdgcn_permlanex16",
     "llvm.amdgcn.qsad.pk.u16.u8" => "__builtin_amdgcn_qsad_pk_u16_u8",
     "llvm.amdgcn.queue.ptr" => "__builtin_amdgcn_queue_ptr",
@@ -122,19 +176,40 @@ match name {
     "llvm.amdgcn.s.setprio" => "__builtin_amdgcn_s_setprio",
     "llvm.amdgcn.s.setreg" => "__builtin_amdgcn_s_setreg",
     "llvm.amdgcn.s.sleep" => "__builtin_amdgcn_s_sleep",
+    "llvm.amdgcn.s.wait.event.export.ready" => "__builtin_amdgcn_s_wait_event_export_ready",
     "llvm.amdgcn.s.waitcnt" => "__builtin_amdgcn_s_waitcnt",
     "llvm.amdgcn.sad.hi.u8" => "__builtin_amdgcn_sad_hi_u8",
     "llvm.amdgcn.sad.u16" => "__builtin_amdgcn_sad_u16",
     "llvm.amdgcn.sad.u8" => "__builtin_amdgcn_sad_u8",
     "llvm.amdgcn.sched.barrier" => "__builtin_amdgcn_sched_barrier",
+    "llvm.amdgcn.sched.group.barrier" => "__builtin_amdgcn_sched_group_barrier",
     "llvm.amdgcn.sdot2" => "__builtin_amdgcn_sdot2",
     "llvm.amdgcn.sdot4" => "__builtin_amdgcn_sdot4",
     "llvm.amdgcn.sdot8" => "__builtin_amdgcn_sdot8",
+    "llvm.amdgcn.smfmac.f32.16x16x32.bf16" => "__builtin_amdgcn_smfmac_f32_16x16x32_bf16",
+    "llvm.amdgcn.smfmac.f32.16x16x32.f16" => "__builtin_amdgcn_smfmac_f32_16x16x32_f16",
+    "llvm.amdgcn.smfmac.f32.16x16x64.bf8.bf8" => "__builtin_amdgcn_smfmac_f32_16x16x64_bf8_bf8",
+    "llvm.amdgcn.smfmac.f32.16x16x64.bf8.fp8" => "__builtin_amdgcn_smfmac_f32_16x16x64_bf8_fp8",
+    "llvm.amdgcn.smfmac.f32.16x16x64.fp8.bf8" => "__builtin_amdgcn_smfmac_f32_16x16x64_fp8_bf8",
+    "llvm.amdgcn.smfmac.f32.16x16x64.fp8.fp8" => "__builtin_amdgcn_smfmac_f32_16x16x64_fp8_fp8",
+    "llvm.amdgcn.smfmac.f32.32x32x16.bf16" => "__builtin_amdgcn_smfmac_f32_32x32x16_bf16",
+    "llvm.amdgcn.smfmac.f32.32x32x16.f16" => "__builtin_amdgcn_smfmac_f32_32x32x16_f16",
+    "llvm.amdgcn.smfmac.f32.32x32x32.bf8.bf8" => "__builtin_amdgcn_smfmac_f32_32x32x32_bf8_bf8",
+    "llvm.amdgcn.smfmac.f32.32x32x32.bf8.fp8" => "__builtin_amdgcn_smfmac_f32_32x32x32_bf8_fp8",
+    "llvm.amdgcn.smfmac.f32.32x32x32.fp8.bf8" => "__builtin_amdgcn_smfmac_f32_32x32x32_fp8_bf8",
+    "llvm.amdgcn.smfmac.f32.32x32x32.fp8.fp8" => "__builtin_amdgcn_smfmac_f32_32x32x32_fp8_fp8",
+    "llvm.amdgcn.smfmac.i32.16x16x64.i8" => "__builtin_amdgcn_smfmac_i32_16x16x64_i8",
+    "llvm.amdgcn.smfmac.i32.32x32x32.i8" => "__builtin_amdgcn_smfmac_i32_32x32x32_i8",
+    "llvm.amdgcn.sudot4" => "__builtin_amdgcn_sudot4",
+    "llvm.amdgcn.sudot8" => "__builtin_amdgcn_sudot8",
     "llvm.amdgcn.udot2" => "__builtin_amdgcn_udot2",
     "llvm.amdgcn.udot4" => "__builtin_amdgcn_udot4",
     "llvm.amdgcn.udot8" => "__builtin_amdgcn_udot8",
     "llvm.amdgcn.wave.barrier" => "__builtin_amdgcn_wave_barrier",
     "llvm.amdgcn.wavefrontsize" => "__builtin_amdgcn_wavefrontsize",
+    "llvm.amdgcn.workgroup.id.x" => "__builtin_amdgcn_workgroup_id_x",
+    "llvm.amdgcn.workgroup.id.y" => "__builtin_amdgcn_workgroup_id_y",
+    "llvm.amdgcn.workgroup.id.z" => "__builtin_amdgcn_workgroup_id_z",
     "llvm.amdgcn.writelane" => "__builtin_amdgcn_writelane",
     // arm
     "llvm.arm.cdp" => "__builtin_arm_cdp",
@@ -249,6 +324,8 @@ match name {
     "llvm.bpf.pseudo" => "__builtin_bpf_pseudo",
     // cuda
     "llvm.cuda.syncthreads" => "__syncthreads",
+    // dx
+    "llvm.dx.create.handle" => "__builtin_hlsl_create_handle",
     // hexagon
     "llvm.hexagon.A2.abs" => "__builtin_HEXAGON_A2_abs",
     "llvm.hexagon.A2.absp" => "__builtin_HEXAGON_A2_absp",
@@ -459,6 +536,11 @@ match name {
     "llvm.hexagon.A4.vrminuw" => "__builtin_HEXAGON_A4_vrminuw",
     "llvm.hexagon.A4.vrminw" => "__builtin_HEXAGON_A4_vrminw",
     "llvm.hexagon.A5.vaddhubs" => "__builtin_HEXAGON_A5_vaddhubs",
+    "llvm.hexagon.A6.vcmpbeq.notany" => "__builtin_HEXAGON_A6_vcmpbeq_notany",
+    "llvm.hexagon.A7.clip" => "__builtin_HEXAGON_A7_clip",
+    "llvm.hexagon.A7.croundd.ri" => "__builtin_HEXAGON_A7_croundd_ri",
+    "llvm.hexagon.A7.croundd.rr" => "__builtin_HEXAGON_A7_croundd_rr",
+    "llvm.hexagon.A7.vclip" => "__builtin_HEXAGON_A7_vclip",
     "llvm.hexagon.C2.all8" => "__builtin_HEXAGON_C2_all8",
     "llvm.hexagon.C2.and" => "__builtin_HEXAGON_C2_and",
     "llvm.hexagon.C2.andn" => "__builtin_HEXAGON_C2_andn",
@@ -557,6 +639,10 @@ match name {
     "llvm.hexagon.F2.dfmax" => "__builtin_HEXAGON_F2_dfmax",
     "llvm.hexagon.F2.dfmin" => "__builtin_HEXAGON_F2_dfmin",
     "llvm.hexagon.F2.dfmpy" => "__builtin_HEXAGON_F2_dfmpy",
+    "llvm.hexagon.F2.dfmpyfix" => "__builtin_HEXAGON_F2_dfmpyfix",
+    "llvm.hexagon.F2.dfmpyhh" => "__builtin_HEXAGON_F2_dfmpyhh",
+    "llvm.hexagon.F2.dfmpylh" => "__builtin_HEXAGON_F2_dfmpylh",
+    "llvm.hexagon.F2.dfmpyll" => "__builtin_HEXAGON_F2_dfmpyll",
     "llvm.hexagon.F2.dfsub" => "__builtin_HEXAGON_F2_dfsub",
     "llvm.hexagon.F2.sfadd" => "__builtin_HEXAGON_F2_sfadd",
     "llvm.hexagon.F2.sfclass" => "__builtin_HEXAGON_F2_sfclass",
@@ -578,6 +664,8 @@ match name {
     "llvm.hexagon.F2.sfmin" => "__builtin_HEXAGON_F2_sfmin",
     "llvm.hexagon.F2.sfmpy" => "__builtin_HEXAGON_F2_sfmpy",
     "llvm.hexagon.F2.sfsub" => "__builtin_HEXAGON_F2_sfsub",
+    "llvm.hexagon.L2.loadw.locked" => "__builtin_HEXAGON_L2_loadw_locked",
+    "llvm.hexagon.L4.loadd.locked" => "__builtin__HEXAGON_L4_loadd_locked",
     "llvm.hexagon.M2.acci" => "__builtin_HEXAGON_M2_acci",
     "llvm.hexagon.M2.accii" => "__builtin_HEXAGON_M2_accii",
     "llvm.hexagon.M2.cmaci.s0" => "__builtin_HEXAGON_M2_cmaci_s0",
@@ -646,6 +734,7 @@ match name {
     "llvm.hexagon.M2.mmpyul.rs1" => "__builtin_HEXAGON_M2_mmpyul_rs1",
     "llvm.hexagon.M2.mmpyul.s0" => "__builtin_HEXAGON_M2_mmpyul_s0",
     "llvm.hexagon.M2.mmpyul.s1" => "__builtin_HEXAGON_M2_mmpyul_s1",
+    "llvm.hexagon.M2.mnaci" => "__builtin_HEXAGON_M2_mnaci",
     "llvm.hexagon.M2.mpy.acc.hh.s0" => "__builtin_HEXAGON_M2_mpy_acc_hh_s0",
     "llvm.hexagon.M2.mpy.acc.hh.s1" => "__builtin_HEXAGON_M2_mpy_acc_hh_s1",
     "llvm.hexagon.M2.mpy.acc.hl.s0" => "__builtin_HEXAGON_M2_mpy_acc_hl_s0",
@@ -894,6 +983,24 @@ match name {
     "llvm.hexagon.M5.vrmpybuu" => "__builtin_HEXAGON_M5_vrmpybuu",
     "llvm.hexagon.M6.vabsdiffb" => "__builtin_HEXAGON_M6_vabsdiffb",
     "llvm.hexagon.M6.vabsdiffub" => "__builtin_HEXAGON_M6_vabsdiffub",
+    "llvm.hexagon.M7.dcmpyiw" => "__builtin_HEXAGON_M7_dcmpyiw",
+    "llvm.hexagon.M7.dcmpyiw.acc" => "__builtin_HEXAGON_M7_dcmpyiw_acc",
+    "llvm.hexagon.M7.dcmpyiwc" => "__builtin_HEXAGON_M7_dcmpyiwc",
+    "llvm.hexagon.M7.dcmpyiwc.acc" => "__builtin_HEXAGON_M7_dcmpyiwc_acc",
+    "llvm.hexagon.M7.dcmpyrw" => "__builtin_HEXAGON_M7_dcmpyrw",
+    "llvm.hexagon.M7.dcmpyrw.acc" => "__builtin_HEXAGON_M7_dcmpyrw_acc",
+    "llvm.hexagon.M7.dcmpyrwc" => "__builtin_HEXAGON_M7_dcmpyrwc",
+    "llvm.hexagon.M7.dcmpyrwc.acc" => "__builtin_HEXAGON_M7_dcmpyrwc_acc",
+    "llvm.hexagon.M7.vdmpy" => "__builtin_HEXAGON_M7_vdmpy",
+    "llvm.hexagon.M7.vdmpy.acc" => "__builtin_HEXAGON_M7_vdmpy_acc",
+    "llvm.hexagon.M7.wcmpyiw" => "__builtin_HEXAGON_M7_wcmpyiw",
+    "llvm.hexagon.M7.wcmpyiw.rnd" => "__builtin_HEXAGON_M7_wcmpyiw_rnd",
+    "llvm.hexagon.M7.wcmpyiwc" => "__builtin_HEXAGON_M7_wcmpyiwc",
+    "llvm.hexagon.M7.wcmpyiwc.rnd" => "__builtin_HEXAGON_M7_wcmpyiwc_rnd",
+    "llvm.hexagon.M7.wcmpyrw" => "__builtin_HEXAGON_M7_wcmpyrw",
+    "llvm.hexagon.M7.wcmpyrw.rnd" => "__builtin_HEXAGON_M7_wcmpyrw_rnd",
+    "llvm.hexagon.M7.wcmpyrwc" => "__builtin_HEXAGON_M7_wcmpyrwc",
+    "llvm.hexagon.M7.wcmpyrwc.rnd" => "__builtin_HEXAGON_M7_wcmpyrwc_rnd",
     "llvm.hexagon.S2.addasl.rrri" => "__builtin_HEXAGON_S2_addasl_rrri",
     "llvm.hexagon.S2.asl.i.p" => "__builtin_HEXAGON_S2_asl_i_p",
     "llvm.hexagon.S2.asl.i.p.acc" => "__builtin_HEXAGON_S2_asl_i_p_acc",
@@ -1023,6 +1130,7 @@ match name {
     "llvm.hexagon.S2.lsr.r.r.or" => "__builtin_HEXAGON_S2_lsr_r_r_or",
     "llvm.hexagon.S2.lsr.r.vh" => "__builtin_HEXAGON_S2_lsr_r_vh",
     "llvm.hexagon.S2.lsr.r.vw" => "__builtin_HEXAGON_S2_lsr_r_vw",
+    "llvm.hexagon.S2.mask" => "__builtin_HEXAGON_S2_mask",
     "llvm.hexagon.S2.packhl" => "__builtin_HEXAGON_S2_packhl",
     "llvm.hexagon.S2.parityp" => "__builtin_HEXAGON_S2_parityp",
     "llvm.hexagon.S2.setbit.i" => "__builtin_HEXAGON_S2_setbit_i",
@@ -1031,6 +1139,12 @@ match name {
     "llvm.hexagon.S2.shuffeh" => "__builtin_HEXAGON_S2_shuffeh",
     "llvm.hexagon.S2.shuffob" => "__builtin_HEXAGON_S2_shuffob",
     "llvm.hexagon.S2.shuffoh" => "__builtin_HEXAGON_S2_shuffoh",
+    "llvm.hexagon.S2.storerb.pbr" => "__builtin_brev_stb",
+    "llvm.hexagon.S2.storerd.pbr" => "__builtin_brev_std",
+    "llvm.hexagon.S2.storerf.pbr" => "__builtin_brev_sthhi",
+    "llvm.hexagon.S2.storerh.pbr" => "__builtin_brev_sth",
+    "llvm.hexagon.S2.storeri.pbr" => "__builtin_brev_stw",
+    "llvm.hexagon.S2.storew.locked" => "__builtin_HEXAGON_S2_storew_locked",
     "llvm.hexagon.S2.svsathb" => "__builtin_HEXAGON_S2_svsathb",
     "llvm.hexagon.S2.svsathub" => "__builtin_HEXAGON_S2_svsathub",
     "llvm.hexagon.S2.tableidxb.goodsyntax" => "__builtin_HEXAGON_S2_tableidxb_goodsyntax",
@@ -1089,6 +1203,7 @@ match name {
     "llvm.hexagon.S4.ori.asl.ri" => "__builtin_HEXAGON_S4_ori_asl_ri",
     "llvm.hexagon.S4.ori.lsr.ri" => "__builtin_HEXAGON_S4_ori_lsr_ri",
     "llvm.hexagon.S4.parity" => "__builtin_HEXAGON_S4_parity",
+    "llvm.hexagon.S4.stored.locked" => "__builtin_HEXAGON_S4_stored_locked",
     "llvm.hexagon.S4.subaddi" => "__builtin_HEXAGON_S4_subaddi",
     "llvm.hexagon.S4.subi.asl.ri" => "__builtin_HEXAGON_S4_subi_asl_ri",
     "llvm.hexagon.S4.subi.lsr.ri" => "__builtin_HEXAGON_S4_subi_lsr_ri",
@@ -1126,8 +1241,56 @@ match name {
     "llvm.hexagon.V6.hi.128B" => "__builtin_HEXAGON_V6_hi_128B",
     "llvm.hexagon.V6.lo" => "__builtin_HEXAGON_V6_lo",
     "llvm.hexagon.V6.lo.128B" => "__builtin_HEXAGON_V6_lo_128B",
+    "llvm.hexagon.V6.lvsplatb" => "__builtin_HEXAGON_V6_lvsplatb",
+    "llvm.hexagon.V6.lvsplatb.128B" => "__builtin_HEXAGON_V6_lvsplatb_128B",
+    "llvm.hexagon.V6.lvsplath" => "__builtin_HEXAGON_V6_lvsplath",
+    "llvm.hexagon.V6.lvsplath.128B" => "__builtin_HEXAGON_V6_lvsplath_128B",
     "llvm.hexagon.V6.lvsplatw" => "__builtin_HEXAGON_V6_lvsplatw",
     "llvm.hexagon.V6.lvsplatw.128B" => "__builtin_HEXAGON_V6_lvsplatw_128B",
+    "llvm.hexagon.V6.pred.and" => "__builtin_HEXAGON_V6_pred_and",
+    "llvm.hexagon.V6.pred.and.128B" => "__builtin_HEXAGON_V6_pred_and_128B",
+    "llvm.hexagon.V6.pred.and.n" => "__builtin_HEXAGON_V6_pred_and_n",
+    "llvm.hexagon.V6.pred.and.n.128B" => "__builtin_HEXAGON_V6_pred_and_n_128B",
+    "llvm.hexagon.V6.pred.not" => "__builtin_HEXAGON_V6_pred_not",
+    "llvm.hexagon.V6.pred.not.128B" => "__builtin_HEXAGON_V6_pred_not_128B",
+    "llvm.hexagon.V6.pred.or" => "__builtin_HEXAGON_V6_pred_or",
+    "llvm.hexagon.V6.pred.or.128B" => "__builtin_HEXAGON_V6_pred_or_128B",
+    "llvm.hexagon.V6.pred.or.n" => "__builtin_HEXAGON_V6_pred_or_n",
+    "llvm.hexagon.V6.pred.or.n.128B" => "__builtin_HEXAGON_V6_pred_or_n_128B",
+    "llvm.hexagon.V6.pred.scalar2" => "__builtin_HEXAGON_V6_pred_scalar2",
+    "llvm.hexagon.V6.pred.scalar2.128B" => "__builtin_HEXAGON_V6_pred_scalar2_128B",
+    "llvm.hexagon.V6.pred.scalar2v2" => "__builtin_HEXAGON_V6_pred_scalar2v2",
+    "llvm.hexagon.V6.pred.scalar2v2.128B" => "__builtin_HEXAGON_V6_pred_scalar2v2_128B",
+    "llvm.hexagon.V6.pred.xor" => "__builtin_HEXAGON_V6_pred_xor",
+    "llvm.hexagon.V6.pred.xor.128B" => "__builtin_HEXAGON_V6_pred_xor_128B",
+    "llvm.hexagon.V6.shuffeqh" => "__builtin_HEXAGON_V6_shuffeqh",
+    "llvm.hexagon.V6.shuffeqh.128B" => "__builtin_HEXAGON_V6_shuffeqh_128B",
+    "llvm.hexagon.V6.shuffeqw" => "__builtin_HEXAGON_V6_shuffeqw",
+    "llvm.hexagon.V6.shuffeqw.128B" => "__builtin_HEXAGON_V6_shuffeqw_128B",
+    "llvm.hexagon.V6.v6mpyhubs10" => "__builtin_HEXAGON_V6_v6mpyhubs10",
+    "llvm.hexagon.V6.v6mpyhubs10.128B" => "__builtin_HEXAGON_V6_v6mpyhubs10_128B",
+    "llvm.hexagon.V6.v6mpyhubs10.vxx" => "__builtin_HEXAGON_V6_v6mpyhubs10_vxx",
+    "llvm.hexagon.V6.v6mpyhubs10.vxx.128B" => "__builtin_HEXAGON_V6_v6mpyhubs10_vxx_128B",
+    "llvm.hexagon.V6.v6mpyvubs10" => "__builtin_HEXAGON_V6_v6mpyvubs10",
+    "llvm.hexagon.V6.v6mpyvubs10.128B" => "__builtin_HEXAGON_V6_v6mpyvubs10_128B",
+    "llvm.hexagon.V6.v6mpyvubs10.vxx" => "__builtin_HEXAGON_V6_v6mpyvubs10_vxx",
+    "llvm.hexagon.V6.v6mpyvubs10.vxx.128B" => "__builtin_HEXAGON_V6_v6mpyvubs10_vxx_128B",
+    "llvm.hexagon.V6.vS32b.nqpred.ai" => "__builtin_HEXAGON_V6_vS32b_nqpred_ai",
+    "llvm.hexagon.V6.vS32b.nqpred.ai.128B" => "__builtin_HEXAGON_V6_vS32b_nqpred_ai_128B",
+    "llvm.hexagon.V6.vS32b.nt.nqpred.ai" => "__builtin_HEXAGON_V6_vS32b_nt_nqpred_ai",
+    "llvm.hexagon.V6.vS32b.nt.nqpred.ai.128B" => "__builtin_HEXAGON_V6_vS32b_nt_nqpred_ai_128B",
+    "llvm.hexagon.V6.vS32b.nt.qpred.ai" => "__builtin_HEXAGON_V6_vS32b_nt_qpred_ai",
+    "llvm.hexagon.V6.vS32b.nt.qpred.ai.128B" => "__builtin_HEXAGON_V6_vS32b_nt_qpred_ai_128B",
+    "llvm.hexagon.V6.vS32b.qpred.ai" => "__builtin_HEXAGON_V6_vS32b_qpred_ai",
+    "llvm.hexagon.V6.vS32b.qpred.ai.128B" => "__builtin_HEXAGON_V6_vS32b_qpred_ai_128B",
+    "llvm.hexagon.V6.vabs.hf" => "__builtin_HEXAGON_V6_vabs_hf",
+    "llvm.hexagon.V6.vabs.hf.128B" => "__builtin_HEXAGON_V6_vabs_hf_128B",
+    "llvm.hexagon.V6.vabs.sf" => "__builtin_HEXAGON_V6_vabs_sf",
+    "llvm.hexagon.V6.vabs.sf.128B" => "__builtin_HEXAGON_V6_vabs_sf_128B",
+    "llvm.hexagon.V6.vabsb" => "__builtin_HEXAGON_V6_vabsb",
+    "llvm.hexagon.V6.vabsb.128B" => "__builtin_HEXAGON_V6_vabsb_128B",
+    "llvm.hexagon.V6.vabsb.sat" => "__builtin_HEXAGON_V6_vabsb_sat",
+    "llvm.hexagon.V6.vabsb.sat.128B" => "__builtin_HEXAGON_V6_vabsb_sat_128B",
     "llvm.hexagon.V6.vabsdiffh" => "__builtin_HEXAGON_V6_vabsdiffh",
     "llvm.hexagon.V6.vabsdiffh.128B" => "__builtin_HEXAGON_V6_vabsdiffh_128B",
     "llvm.hexagon.V6.vabsdiffub" => "__builtin_HEXAGON_V6_vabsdiffub",
@@ -1144,36 +1307,90 @@ match name {
     "llvm.hexagon.V6.vabsw.128B" => "__builtin_HEXAGON_V6_vabsw_128B",
     "llvm.hexagon.V6.vabsw.sat" => "__builtin_HEXAGON_V6_vabsw_sat",
     "llvm.hexagon.V6.vabsw.sat.128B" => "__builtin_HEXAGON_V6_vabsw_sat_128B",
+    "llvm.hexagon.V6.vadd.hf" => "__builtin_HEXAGON_V6_vadd_hf",
+    "llvm.hexagon.V6.vadd.hf.128B" => "__builtin_HEXAGON_V6_vadd_hf_128B",
+    "llvm.hexagon.V6.vadd.hf.hf" => "__builtin_HEXAGON_V6_vadd_hf_hf",
+    "llvm.hexagon.V6.vadd.hf.hf.128B" => "__builtin_HEXAGON_V6_vadd_hf_hf_128B",
+    "llvm.hexagon.V6.vadd.qf16" => "__builtin_HEXAGON_V6_vadd_qf16",
+    "llvm.hexagon.V6.vadd.qf16.128B" => "__builtin_HEXAGON_V6_vadd_qf16_128B",
+    "llvm.hexagon.V6.vadd.qf16.mix" => "__builtin_HEXAGON_V6_vadd_qf16_mix",
+    "llvm.hexagon.V6.vadd.qf16.mix.128B" => "__builtin_HEXAGON_V6_vadd_qf16_mix_128B",
+    "llvm.hexagon.V6.vadd.qf32" => "__builtin_HEXAGON_V6_vadd_qf32",
+    "llvm.hexagon.V6.vadd.qf32.128B" => "__builtin_HEXAGON_V6_vadd_qf32_128B",
+    "llvm.hexagon.V6.vadd.qf32.mix" => "__builtin_HEXAGON_V6_vadd_qf32_mix",
+    "llvm.hexagon.V6.vadd.qf32.mix.128B" => "__builtin_HEXAGON_V6_vadd_qf32_mix_128B",
+    "llvm.hexagon.V6.vadd.sf" => "__builtin_HEXAGON_V6_vadd_sf",
+    "llvm.hexagon.V6.vadd.sf.128B" => "__builtin_HEXAGON_V6_vadd_sf_128B",
+    "llvm.hexagon.V6.vadd.sf.bf" => "__builtin_HEXAGON_V6_vadd_sf_bf",
+    "llvm.hexagon.V6.vadd.sf.bf.128B" => "__builtin_HEXAGON_V6_vadd_sf_bf_128B",
+    "llvm.hexagon.V6.vadd.sf.hf" => "__builtin_HEXAGON_V6_vadd_sf_hf",
+    "llvm.hexagon.V6.vadd.sf.hf.128B" => "__builtin_HEXAGON_V6_vadd_sf_hf_128B",
+    "llvm.hexagon.V6.vadd.sf.sf" => "__builtin_HEXAGON_V6_vadd_sf_sf",
+    "llvm.hexagon.V6.vadd.sf.sf.128B" => "__builtin_HEXAGON_V6_vadd_sf_sf_128B",
     "llvm.hexagon.V6.vaddb" => "__builtin_HEXAGON_V6_vaddb",
     "llvm.hexagon.V6.vaddb.128B" => "__builtin_HEXAGON_V6_vaddb_128B",
     "llvm.hexagon.V6.vaddb.dv" => "__builtin_HEXAGON_V6_vaddb_dv",
     "llvm.hexagon.V6.vaddb.dv.128B" => "__builtin_HEXAGON_V6_vaddb_dv_128B",
+    "llvm.hexagon.V6.vaddbnq" => "__builtin_HEXAGON_V6_vaddbnq",
+    "llvm.hexagon.V6.vaddbnq.128B" => "__builtin_HEXAGON_V6_vaddbnq_128B",
+    "llvm.hexagon.V6.vaddbq" => "__builtin_HEXAGON_V6_vaddbq",
+    "llvm.hexagon.V6.vaddbq.128B" => "__builtin_HEXAGON_V6_vaddbq_128B",
+    "llvm.hexagon.V6.vaddbsat" => "__builtin_HEXAGON_V6_vaddbsat",
+    "llvm.hexagon.V6.vaddbsat.128B" => "__builtin_HEXAGON_V6_vaddbsat_128B",
+    "llvm.hexagon.V6.vaddbsat.dv" => "__builtin_HEXAGON_V6_vaddbsat_dv",
+    "llvm.hexagon.V6.vaddbsat.dv.128B" => "__builtin_HEXAGON_V6_vaddbsat_dv_128B",
+    "llvm.hexagon.V6.vaddcarrysat" => "__builtin_HEXAGON_V6_vaddcarrysat",
+    "llvm.hexagon.V6.vaddcarrysat.128B" => "__builtin_HEXAGON_V6_vaddcarrysat_128B",
+    "llvm.hexagon.V6.vaddclbh" => "__builtin_HEXAGON_V6_vaddclbh",
+    "llvm.hexagon.V6.vaddclbh.128B" => "__builtin_HEXAGON_V6_vaddclbh_128B",
+    "llvm.hexagon.V6.vaddclbw" => "__builtin_HEXAGON_V6_vaddclbw",
+    "llvm.hexagon.V6.vaddclbw.128B" => "__builtin_HEXAGON_V6_vaddclbw_128B",
     "llvm.hexagon.V6.vaddh" => "__builtin_HEXAGON_V6_vaddh",
     "llvm.hexagon.V6.vaddh.128B" => "__builtin_HEXAGON_V6_vaddh_128B",
     "llvm.hexagon.V6.vaddh.dv" => "__builtin_HEXAGON_V6_vaddh_dv",
     "llvm.hexagon.V6.vaddh.dv.128B" => "__builtin_HEXAGON_V6_vaddh_dv_128B",
+    "llvm.hexagon.V6.vaddhnq" => "__builtin_HEXAGON_V6_vaddhnq",
+    "llvm.hexagon.V6.vaddhnq.128B" => "__builtin_HEXAGON_V6_vaddhnq_128B",
+    "llvm.hexagon.V6.vaddhq" => "__builtin_HEXAGON_V6_vaddhq",
+    "llvm.hexagon.V6.vaddhq.128B" => "__builtin_HEXAGON_V6_vaddhq_128B",
     "llvm.hexagon.V6.vaddhsat" => "__builtin_HEXAGON_V6_vaddhsat",
     "llvm.hexagon.V6.vaddhsat.128B" => "__builtin_HEXAGON_V6_vaddhsat_128B",
     "llvm.hexagon.V6.vaddhsat.dv" => "__builtin_HEXAGON_V6_vaddhsat_dv",
     "llvm.hexagon.V6.vaddhsat.dv.128B" => "__builtin_HEXAGON_V6_vaddhsat_dv_128B",
     "llvm.hexagon.V6.vaddhw" => "__builtin_HEXAGON_V6_vaddhw",
     "llvm.hexagon.V6.vaddhw.128B" => "__builtin_HEXAGON_V6_vaddhw_128B",
+    "llvm.hexagon.V6.vaddhw.acc" => "__builtin_HEXAGON_V6_vaddhw_acc",
+    "llvm.hexagon.V6.vaddhw.acc.128B" => "__builtin_HEXAGON_V6_vaddhw_acc_128B",
     "llvm.hexagon.V6.vaddubh" => "__builtin_HEXAGON_V6_vaddubh",
     "llvm.hexagon.V6.vaddubh.128B" => "__builtin_HEXAGON_V6_vaddubh_128B",
+    "llvm.hexagon.V6.vaddubh.acc" => "__builtin_HEXAGON_V6_vaddubh_acc",
+    "llvm.hexagon.V6.vaddubh.acc.128B" => "__builtin_HEXAGON_V6_vaddubh_acc_128B",
     "llvm.hexagon.V6.vaddubsat" => "__builtin_HEXAGON_V6_vaddubsat",
     "llvm.hexagon.V6.vaddubsat.128B" => "__builtin_HEXAGON_V6_vaddubsat_128B",
     "llvm.hexagon.V6.vaddubsat.dv" => "__builtin_HEXAGON_V6_vaddubsat_dv",
     "llvm.hexagon.V6.vaddubsat.dv.128B" => "__builtin_HEXAGON_V6_vaddubsat_dv_128B",
+    "llvm.hexagon.V6.vaddububb.sat" => "__builtin_HEXAGON_V6_vaddububb_sat",
+    "llvm.hexagon.V6.vaddububb.sat.128B" => "__builtin_HEXAGON_V6_vaddububb_sat_128B",
     "llvm.hexagon.V6.vadduhsat" => "__builtin_HEXAGON_V6_vadduhsat",
     "llvm.hexagon.V6.vadduhsat.128B" => "__builtin_HEXAGON_V6_vadduhsat_128B",
     "llvm.hexagon.V6.vadduhsat.dv" => "__builtin_HEXAGON_V6_vadduhsat_dv",
     "llvm.hexagon.V6.vadduhsat.dv.128B" => "__builtin_HEXAGON_V6_vadduhsat_dv_128B",
     "llvm.hexagon.V6.vadduhw" => "__builtin_HEXAGON_V6_vadduhw",
     "llvm.hexagon.V6.vadduhw.128B" => "__builtin_HEXAGON_V6_vadduhw_128B",
+    "llvm.hexagon.V6.vadduhw.acc" => "__builtin_HEXAGON_V6_vadduhw_acc",
+    "llvm.hexagon.V6.vadduhw.acc.128B" => "__builtin_HEXAGON_V6_vadduhw_acc_128B",
+    "llvm.hexagon.V6.vadduwsat" => "__builtin_HEXAGON_V6_vadduwsat",
+    "llvm.hexagon.V6.vadduwsat.128B" => "__builtin_HEXAGON_V6_vadduwsat_128B",
+    "llvm.hexagon.V6.vadduwsat.dv" => "__builtin_HEXAGON_V6_vadduwsat_dv",
+    "llvm.hexagon.V6.vadduwsat.dv.128B" => "__builtin_HEXAGON_V6_vadduwsat_dv_128B",
     "llvm.hexagon.V6.vaddw" => "__builtin_HEXAGON_V6_vaddw",
     "llvm.hexagon.V6.vaddw.128B" => "__builtin_HEXAGON_V6_vaddw_128B",
     "llvm.hexagon.V6.vaddw.dv" => "__builtin_HEXAGON_V6_vaddw_dv",
     "llvm.hexagon.V6.vaddw.dv.128B" => "__builtin_HEXAGON_V6_vaddw_dv_128B",
+    "llvm.hexagon.V6.vaddwnq" => "__builtin_HEXAGON_V6_vaddwnq",
+    "llvm.hexagon.V6.vaddwnq.128B" => "__builtin_HEXAGON_V6_vaddwnq_128B",
+    "llvm.hexagon.V6.vaddwq" => "__builtin_HEXAGON_V6_vaddwq",
+    "llvm.hexagon.V6.vaddwq.128B" => "__builtin_HEXAGON_V6_vaddwq_128B",
     "llvm.hexagon.V6.vaddwsat" => "__builtin_HEXAGON_V6_vaddwsat",
     "llvm.hexagon.V6.vaddwsat.128B" => "__builtin_HEXAGON_V6_vaddwsat_128B",
     "llvm.hexagon.V6.vaddwsat.dv" => "__builtin_HEXAGON_V6_vaddwsat_dv",
@@ -1184,8 +1401,26 @@ match name {
     "llvm.hexagon.V6.valignbi.128B" => "__builtin_HEXAGON_V6_valignbi_128B",
     "llvm.hexagon.V6.vand" => "__builtin_HEXAGON_V6_vand",
     "llvm.hexagon.V6.vand.128B" => "__builtin_HEXAGON_V6_vand_128B",
+    "llvm.hexagon.V6.vandnqrt" => "__builtin_HEXAGON_V6_vandnqrt",
+    "llvm.hexagon.V6.vandnqrt.128B" => "__builtin_HEXAGON_V6_vandnqrt_128B",
+    "llvm.hexagon.V6.vandnqrt.acc" => "__builtin_HEXAGON_V6_vandnqrt_acc",
+    "llvm.hexagon.V6.vandnqrt.acc.128B" => "__builtin_HEXAGON_V6_vandnqrt_acc_128B",
+    "llvm.hexagon.V6.vandqrt" => "__builtin_HEXAGON_V6_vandqrt",
+    "llvm.hexagon.V6.vandqrt.128B" => "__builtin_HEXAGON_V6_vandqrt_128B",
+    "llvm.hexagon.V6.vandqrt.acc" => "__builtin_HEXAGON_V6_vandqrt_acc",
+    "llvm.hexagon.V6.vandqrt.acc.128B" => "__builtin_HEXAGON_V6_vandqrt_acc_128B",
+    "llvm.hexagon.V6.vandvnqv" => "__builtin_HEXAGON_V6_vandvnqv",
+    "llvm.hexagon.V6.vandvnqv.128B" => "__builtin_HEXAGON_V6_vandvnqv_128B",
+    "llvm.hexagon.V6.vandvqv" => "__builtin_HEXAGON_V6_vandvqv",
+    "llvm.hexagon.V6.vandvqv.128B" => "__builtin_HEXAGON_V6_vandvqv_128B",
+    "llvm.hexagon.V6.vandvrt" => "__builtin_HEXAGON_V6_vandvrt",
+    "llvm.hexagon.V6.vandvrt.128B" => "__builtin_HEXAGON_V6_vandvrt_128B",
+    "llvm.hexagon.V6.vandvrt.acc" => "__builtin_HEXAGON_V6_vandvrt_acc",
+    "llvm.hexagon.V6.vandvrt.acc.128B" => "__builtin_HEXAGON_V6_vandvrt_acc_128B",
     "llvm.hexagon.V6.vaslh" => "__builtin_HEXAGON_V6_vaslh",
     "llvm.hexagon.V6.vaslh.128B" => "__builtin_HEXAGON_V6_vaslh_128B",
+    "llvm.hexagon.V6.vaslh.acc" => "__builtin_HEXAGON_V6_vaslh_acc",
+    "llvm.hexagon.V6.vaslh.acc.128B" => "__builtin_HEXAGON_V6_vaslh_acc_128B",
     "llvm.hexagon.V6.vaslhv" => "__builtin_HEXAGON_V6_vaslhv",
     "llvm.hexagon.V6.vaslhv.128B" => "__builtin_HEXAGON_V6_vaslhv_128B",
     "llvm.hexagon.V6.vaslw" => "__builtin_HEXAGON_V6_vaslw",
@@ -1194,16 +1429,38 @@ match name {
     "llvm.hexagon.V6.vaslw.acc.128B" => "__builtin_HEXAGON_V6_vaslw_acc_128B",
     "llvm.hexagon.V6.vaslwv" => "__builtin_HEXAGON_V6_vaslwv",
     "llvm.hexagon.V6.vaslwv.128B" => "__builtin_HEXAGON_V6_vaslwv_128B",
+    "llvm.hexagon.V6.vasr.into" => "__builtin_HEXAGON_V6_vasr_into",
+    "llvm.hexagon.V6.vasr.into.128B" => "__builtin_HEXAGON_V6_vasr_into_128B",
     "llvm.hexagon.V6.vasrh" => "__builtin_HEXAGON_V6_vasrh",
     "llvm.hexagon.V6.vasrh.128B" => "__builtin_HEXAGON_V6_vasrh_128B",
+    "llvm.hexagon.V6.vasrh.acc" => "__builtin_HEXAGON_V6_vasrh_acc",
+    "llvm.hexagon.V6.vasrh.acc.128B" => "__builtin_HEXAGON_V6_vasrh_acc_128B",
     "llvm.hexagon.V6.vasrhbrndsat" => "__builtin_HEXAGON_V6_vasrhbrndsat",
     "llvm.hexagon.V6.vasrhbrndsat.128B" => "__builtin_HEXAGON_V6_vasrhbrndsat_128B",
+    "llvm.hexagon.V6.vasrhbsat" => "__builtin_HEXAGON_V6_vasrhbsat",
+    "llvm.hexagon.V6.vasrhbsat.128B" => "__builtin_HEXAGON_V6_vasrhbsat_128B",
     "llvm.hexagon.V6.vasrhubrndsat" => "__builtin_HEXAGON_V6_vasrhubrndsat",
     "llvm.hexagon.V6.vasrhubrndsat.128B" => "__builtin_HEXAGON_V6_vasrhubrndsat_128B",
     "llvm.hexagon.V6.vasrhubsat" => "__builtin_HEXAGON_V6_vasrhubsat",
     "llvm.hexagon.V6.vasrhubsat.128B" => "__builtin_HEXAGON_V6_vasrhubsat_128B",
     "llvm.hexagon.V6.vasrhv" => "__builtin_HEXAGON_V6_vasrhv",
     "llvm.hexagon.V6.vasrhv.128B" => "__builtin_HEXAGON_V6_vasrhv_128B",
+    "llvm.hexagon.V6.vasruhubrndsat" => "__builtin_HEXAGON_V6_vasruhubrndsat",
+    "llvm.hexagon.V6.vasruhubrndsat.128B" => "__builtin_HEXAGON_V6_vasruhubrndsat_128B",
+    "llvm.hexagon.V6.vasruhubsat" => "__builtin_HEXAGON_V6_vasruhubsat",
+    "llvm.hexagon.V6.vasruhubsat.128B" => "__builtin_HEXAGON_V6_vasruhubsat_128B",
+    "llvm.hexagon.V6.vasruwuhrndsat" => "__builtin_HEXAGON_V6_vasruwuhrndsat",
+    "llvm.hexagon.V6.vasruwuhrndsat.128B" => "__builtin_HEXAGON_V6_vasruwuhrndsat_128B",
+    "llvm.hexagon.V6.vasruwuhsat" => "__builtin_HEXAGON_V6_vasruwuhsat",
+    "llvm.hexagon.V6.vasruwuhsat.128B" => "__builtin_HEXAGON_V6_vasruwuhsat_128B",
+    "llvm.hexagon.V6.vasrvuhubrndsat" => "__builtin_HEXAGON_V6_vasrvuhubrndsat",
+    "llvm.hexagon.V6.vasrvuhubrndsat.128B" => "__builtin_HEXAGON_V6_vasrvuhubrndsat_128B",
+    "llvm.hexagon.V6.vasrvuhubsat" => "__builtin_HEXAGON_V6_vasrvuhubsat",
+    "llvm.hexagon.V6.vasrvuhubsat.128B" => "__builtin_HEXAGON_V6_vasrvuhubsat_128B",
+    "llvm.hexagon.V6.vasrvwuhrndsat" => "__builtin_HEXAGON_V6_vasrvwuhrndsat",
+    "llvm.hexagon.V6.vasrvwuhrndsat.128B" => "__builtin_HEXAGON_V6_vasrvwuhrndsat_128B",
+    "llvm.hexagon.V6.vasrvwuhsat" => "__builtin_HEXAGON_V6_vasrvwuhsat",
+    "llvm.hexagon.V6.vasrvwuhsat.128B" => "__builtin_HEXAGON_V6_vasrvwuhsat_128B",
     "llvm.hexagon.V6.vasrw" => "__builtin_HEXAGON_V6_vasrw",
     "llvm.hexagon.V6.vasrw.128B" => "__builtin_HEXAGON_V6_vasrw_128B",
     "llvm.hexagon.V6.vasrw.acc" => "__builtin_HEXAGON_V6_vasrw_acc",
@@ -1214,14 +1471,22 @@ match name {
     "llvm.hexagon.V6.vasrwhrndsat.128B" => "__builtin_HEXAGON_V6_vasrwhrndsat_128B",
     "llvm.hexagon.V6.vasrwhsat" => "__builtin_HEXAGON_V6_vasrwhsat",
     "llvm.hexagon.V6.vasrwhsat.128B" => "__builtin_HEXAGON_V6_vasrwhsat_128B",
+    "llvm.hexagon.V6.vasrwuhrndsat" => "__builtin_HEXAGON_V6_vasrwuhrndsat",
+    "llvm.hexagon.V6.vasrwuhrndsat.128B" => "__builtin_HEXAGON_V6_vasrwuhrndsat_128B",
     "llvm.hexagon.V6.vasrwuhsat" => "__builtin_HEXAGON_V6_vasrwuhsat",
     "llvm.hexagon.V6.vasrwuhsat.128B" => "__builtin_HEXAGON_V6_vasrwuhsat_128B",
     "llvm.hexagon.V6.vasrwv" => "__builtin_HEXAGON_V6_vasrwv",
     "llvm.hexagon.V6.vasrwv.128B" => "__builtin_HEXAGON_V6_vasrwv_128B",
     "llvm.hexagon.V6.vassign" => "__builtin_HEXAGON_V6_vassign",
     "llvm.hexagon.V6.vassign.128B" => "__builtin_HEXAGON_V6_vassign_128B",
+    "llvm.hexagon.V6.vassign.fp" => "__builtin_HEXAGON_V6_vassign_fp",
+    "llvm.hexagon.V6.vassign.fp.128B" => "__builtin_HEXAGON_V6_vassign_fp_128B",
     "llvm.hexagon.V6.vassignp" => "__builtin_HEXAGON_V6_vassignp",
     "llvm.hexagon.V6.vassignp.128B" => "__builtin_HEXAGON_V6_vassignp_128B",
+    "llvm.hexagon.V6.vavgb" => "__builtin_HEXAGON_V6_vavgb",
+    "llvm.hexagon.V6.vavgb.128B" => "__builtin_HEXAGON_V6_vavgb_128B",
+    "llvm.hexagon.V6.vavgbrnd" => "__builtin_HEXAGON_V6_vavgbrnd",
+    "llvm.hexagon.V6.vavgbrnd.128B" => "__builtin_HEXAGON_V6_vavgbrnd_128B",
     "llvm.hexagon.V6.vavgh" => "__builtin_HEXAGON_V6_vavgh",
     "llvm.hexagon.V6.vavgh.128B" => "__builtin_HEXAGON_V6_vavgh_128B",
     "llvm.hexagon.V6.vavghrnd" => "__builtin_HEXAGON_V6_vavghrnd",
@@ -1234,6 +1499,10 @@ match name {
     "llvm.hexagon.V6.vavguh.128B" => "__builtin_HEXAGON_V6_vavguh_128B",
     "llvm.hexagon.V6.vavguhrnd" => "__builtin_HEXAGON_V6_vavguhrnd",
     "llvm.hexagon.V6.vavguhrnd.128B" => "__builtin_HEXAGON_V6_vavguhrnd_128B",
+    "llvm.hexagon.V6.vavguw" => "__builtin_HEXAGON_V6_vavguw",
+    "llvm.hexagon.V6.vavguw.128B" => "__builtin_HEXAGON_V6_vavguw_128B",
+    "llvm.hexagon.V6.vavguwrnd" => "__builtin_HEXAGON_V6_vavguwrnd",
+    "llvm.hexagon.V6.vavguwrnd.128B" => "__builtin_HEXAGON_V6_vavguwrnd_128B",
     "llvm.hexagon.V6.vavgw" => "__builtin_HEXAGON_V6_vavgw",
     "llvm.hexagon.V6.vavgw.128B" => "__builtin_HEXAGON_V6_vavgw_128B",
     "llvm.hexagon.V6.vavgwrnd" => "__builtin_HEXAGON_V6_vavgwrnd",
@@ -1244,8 +1513,46 @@ match name {
     "llvm.hexagon.V6.vcl0w.128B" => "__builtin_HEXAGON_V6_vcl0w_128B",
     "llvm.hexagon.V6.vcombine" => "__builtin_HEXAGON_V6_vcombine",
     "llvm.hexagon.V6.vcombine.128B" => "__builtin_HEXAGON_V6_vcombine_128B",
+    "llvm.hexagon.V6.vconv.h.hf" => "__builtin_HEXAGON_V6_vconv_h_hf",
+    "llvm.hexagon.V6.vconv.h.hf.128B" => "__builtin_HEXAGON_V6_vconv_h_hf_128B",
+    "llvm.hexagon.V6.vconv.hf.h" => "__builtin_HEXAGON_V6_vconv_hf_h",
+    "llvm.hexagon.V6.vconv.hf.h.128B" => "__builtin_HEXAGON_V6_vconv_hf_h_128B",
+    "llvm.hexagon.V6.vconv.hf.qf16" => "__builtin_HEXAGON_V6_vconv_hf_qf16",
+    "llvm.hexagon.V6.vconv.hf.qf16.128B" => "__builtin_HEXAGON_V6_vconv_hf_qf16_128B",
+    "llvm.hexagon.V6.vconv.hf.qf32" => "__builtin_HEXAGON_V6_vconv_hf_qf32",
+    "llvm.hexagon.V6.vconv.hf.qf32.128B" => "__builtin_HEXAGON_V6_vconv_hf_qf32_128B",
+    "llvm.hexagon.V6.vconv.sf.qf32" => "__builtin_HEXAGON_V6_vconv_sf_qf32",
+    "llvm.hexagon.V6.vconv.sf.qf32.128B" => "__builtin_HEXAGON_V6_vconv_sf_qf32_128B",
+    "llvm.hexagon.V6.vconv.sf.w" => "__builtin_HEXAGON_V6_vconv_sf_w",
+    "llvm.hexagon.V6.vconv.sf.w.128B" => "__builtin_HEXAGON_V6_vconv_sf_w_128B",
+    "llvm.hexagon.V6.vconv.w.sf" => "__builtin_HEXAGON_V6_vconv_w_sf",
+    "llvm.hexagon.V6.vconv.w.sf.128B" => "__builtin_HEXAGON_V6_vconv_w_sf_128B",
+    "llvm.hexagon.V6.vcvt.b.hf" => "__builtin_HEXAGON_V6_vcvt_b_hf",
+    "llvm.hexagon.V6.vcvt.b.hf.128B" => "__builtin_HEXAGON_V6_vcvt_b_hf_128B",
+    "llvm.hexagon.V6.vcvt.bf.sf" => "__builtin_HEXAGON_V6_vcvt_bf_sf",
+    "llvm.hexagon.V6.vcvt.bf.sf.128B" => "__builtin_HEXAGON_V6_vcvt_bf_sf_128B",
+    "llvm.hexagon.V6.vcvt.h.hf" => "__builtin_HEXAGON_V6_vcvt_h_hf",
+    "llvm.hexagon.V6.vcvt.h.hf.128B" => "__builtin_HEXAGON_V6_vcvt_h_hf_128B",
+    "llvm.hexagon.V6.vcvt.hf.b" => "__builtin_HEXAGON_V6_vcvt_hf_b",
+    "llvm.hexagon.V6.vcvt.hf.b.128B" => "__builtin_HEXAGON_V6_vcvt_hf_b_128B",
+    "llvm.hexagon.V6.vcvt.hf.h" => "__builtin_HEXAGON_V6_vcvt_hf_h",
+    "llvm.hexagon.V6.vcvt.hf.h.128B" => "__builtin_HEXAGON_V6_vcvt_hf_h_128B",
+    "llvm.hexagon.V6.vcvt.hf.sf" => "__builtin_HEXAGON_V6_vcvt_hf_sf",
+    "llvm.hexagon.V6.vcvt.hf.sf.128B" => "__builtin_HEXAGON_V6_vcvt_hf_sf_128B",
+    "llvm.hexagon.V6.vcvt.hf.ub" => "__builtin_HEXAGON_V6_vcvt_hf_ub",
+    "llvm.hexagon.V6.vcvt.hf.ub.128B" => "__builtin_HEXAGON_V6_vcvt_hf_ub_128B",
+    "llvm.hexagon.V6.vcvt.hf.uh" => "__builtin_HEXAGON_V6_vcvt_hf_uh",
+    "llvm.hexagon.V6.vcvt.hf.uh.128B" => "__builtin_HEXAGON_V6_vcvt_hf_uh_128B",
+    "llvm.hexagon.V6.vcvt.sf.hf" => "__builtin_HEXAGON_V6_vcvt_sf_hf",
+    "llvm.hexagon.V6.vcvt.sf.hf.128B" => "__builtin_HEXAGON_V6_vcvt_sf_hf_128B",
+    "llvm.hexagon.V6.vcvt.ub.hf" => "__builtin_HEXAGON_V6_vcvt_ub_hf",
+    "llvm.hexagon.V6.vcvt.ub.hf.128B" => "__builtin_HEXAGON_V6_vcvt_ub_hf_128B",
+    "llvm.hexagon.V6.vcvt.uh.hf" => "__builtin_HEXAGON_V6_vcvt_uh_hf",
+    "llvm.hexagon.V6.vcvt.uh.hf.128B" => "__builtin_HEXAGON_V6_vcvt_uh_hf_128B",
     "llvm.hexagon.V6.vd0" => "__builtin_HEXAGON_V6_vd0",
     "llvm.hexagon.V6.vd0.128B" => "__builtin_HEXAGON_V6_vd0_128B",
+    "llvm.hexagon.V6.vdd0" => "__builtin_HEXAGON_V6_vdd0",
+    "llvm.hexagon.V6.vdd0.128B" => "__builtin_HEXAGON_V6_vdd0_128B",
     "llvm.hexagon.V6.vdealb" => "__builtin_HEXAGON_V6_vdealb",
     "llvm.hexagon.V6.vdealb.128B" => "__builtin_HEXAGON_V6_vdealb_128B",
     "llvm.hexagon.V6.vdealb4w" => "__builtin_HEXAGON_V6_vdealb4w",
@@ -1256,6 +1563,10 @@ match name {
     "llvm.hexagon.V6.vdealvdd.128B" => "__builtin_HEXAGON_V6_vdealvdd_128B",
     "llvm.hexagon.V6.vdelta" => "__builtin_HEXAGON_V6_vdelta",
     "llvm.hexagon.V6.vdelta.128B" => "__builtin_HEXAGON_V6_vdelta_128B",
+    "llvm.hexagon.V6.vdmpy.sf.hf" => "__builtin_HEXAGON_V6_vdmpy_sf_hf",
+    "llvm.hexagon.V6.vdmpy.sf.hf.128B" => "__builtin_HEXAGON_V6_vdmpy_sf_hf_128B",
+    "llvm.hexagon.V6.vdmpy.sf.hf.acc" => "__builtin_HEXAGON_V6_vdmpy_sf_hf_acc",
+    "llvm.hexagon.V6.vdmpy.sf.hf.acc.128B" => "__builtin_HEXAGON_V6_vdmpy_sf_hf_acc_128B",
     "llvm.hexagon.V6.vdmpybus" => "__builtin_HEXAGON_V6_vdmpybus",
     "llvm.hexagon.V6.vdmpybus.128B" => "__builtin_HEXAGON_V6_vdmpybus_128B",
     "llvm.hexagon.V6.vdmpybus.acc" => "__builtin_HEXAGON_V6_vdmpybus_acc",
@@ -1296,12 +1607,134 @@ match name {
     "llvm.hexagon.V6.vdsaduh.128B" => "__builtin_HEXAGON_V6_vdsaduh_128B",
     "llvm.hexagon.V6.vdsaduh.acc" => "__builtin_HEXAGON_V6_vdsaduh_acc",
     "llvm.hexagon.V6.vdsaduh.acc.128B" => "__builtin_HEXAGON_V6_vdsaduh_acc_128B",
+    "llvm.hexagon.V6.veqb" => "__builtin_HEXAGON_V6_veqb",
+    "llvm.hexagon.V6.veqb.128B" => "__builtin_HEXAGON_V6_veqb_128B",
+    "llvm.hexagon.V6.veqb.and" => "__builtin_HEXAGON_V6_veqb_and",
+    "llvm.hexagon.V6.veqb.and.128B" => "__builtin_HEXAGON_V6_veqb_and_128B",
+    "llvm.hexagon.V6.veqb.or" => "__builtin_HEXAGON_V6_veqb_or",
+    "llvm.hexagon.V6.veqb.or.128B" => "__builtin_HEXAGON_V6_veqb_or_128B",
+    "llvm.hexagon.V6.veqb.xor" => "__builtin_HEXAGON_V6_veqb_xor",
+    "llvm.hexagon.V6.veqb.xor.128B" => "__builtin_HEXAGON_V6_veqb_xor_128B",
+    "llvm.hexagon.V6.veqh" => "__builtin_HEXAGON_V6_veqh",
+    "llvm.hexagon.V6.veqh.128B" => "__builtin_HEXAGON_V6_veqh_128B",
+    "llvm.hexagon.V6.veqh.and" => "__builtin_HEXAGON_V6_veqh_and",
+    "llvm.hexagon.V6.veqh.and.128B" => "__builtin_HEXAGON_V6_veqh_and_128B",
+    "llvm.hexagon.V6.veqh.or" => "__builtin_HEXAGON_V6_veqh_or",
+    "llvm.hexagon.V6.veqh.or.128B" => "__builtin_HEXAGON_V6_veqh_or_128B",
+    "llvm.hexagon.V6.veqh.xor" => "__builtin_HEXAGON_V6_veqh_xor",
+    "llvm.hexagon.V6.veqh.xor.128B" => "__builtin_HEXAGON_V6_veqh_xor_128B",
+    "llvm.hexagon.V6.veqw" => "__builtin_HEXAGON_V6_veqw",
+    "llvm.hexagon.V6.veqw.128B" => "__builtin_HEXAGON_V6_veqw_128B",
+    "llvm.hexagon.V6.veqw.and" => "__builtin_HEXAGON_V6_veqw_and",
+    "llvm.hexagon.V6.veqw.and.128B" => "__builtin_HEXAGON_V6_veqw_and_128B",
+    "llvm.hexagon.V6.veqw.or" => "__builtin_HEXAGON_V6_veqw_or",
+    "llvm.hexagon.V6.veqw.or.128B" => "__builtin_HEXAGON_V6_veqw_or_128B",
+    "llvm.hexagon.V6.veqw.xor" => "__builtin_HEXAGON_V6_veqw_xor",
+    "llvm.hexagon.V6.veqw.xor.128B" => "__builtin_HEXAGON_V6_veqw_xor_128B",
+    "llvm.hexagon.V6.vfmax.hf" => "__builtin_HEXAGON_V6_vfmax_hf",
+    "llvm.hexagon.V6.vfmax.hf.128B" => "__builtin_HEXAGON_V6_vfmax_hf_128B",
+    "llvm.hexagon.V6.vfmax.sf" => "__builtin_HEXAGON_V6_vfmax_sf",
+    "llvm.hexagon.V6.vfmax.sf.128B" => "__builtin_HEXAGON_V6_vfmax_sf_128B",
+    "llvm.hexagon.V6.vfmin.hf" => "__builtin_HEXAGON_V6_vfmin_hf",
+    "llvm.hexagon.V6.vfmin.hf.128B" => "__builtin_HEXAGON_V6_vfmin_hf_128B",
+    "llvm.hexagon.V6.vfmin.sf" => "__builtin_HEXAGON_V6_vfmin_sf",
+    "llvm.hexagon.V6.vfmin.sf.128B" => "__builtin_HEXAGON_V6_vfmin_sf_128B",
+    "llvm.hexagon.V6.vfneg.hf" => "__builtin_HEXAGON_V6_vfneg_hf",
+    "llvm.hexagon.V6.vfneg.hf.128B" => "__builtin_HEXAGON_V6_vfneg_hf_128B",
+    "llvm.hexagon.V6.vfneg.sf" => "__builtin_HEXAGON_V6_vfneg_sf",
+    "llvm.hexagon.V6.vfneg.sf.128B" => "__builtin_HEXAGON_V6_vfneg_sf_128B",
+    "llvm.hexagon.V6.vgathermh" => "__builtin_HEXAGON_V6_vgathermh",
+    "llvm.hexagon.V6.vgathermh.128B" => "__builtin_HEXAGON_V6_vgathermh_128B",
+    "llvm.hexagon.V6.vgathermhq" => "__builtin_HEXAGON_V6_vgathermhq",
+    "llvm.hexagon.V6.vgathermhq.128B" => "__builtin_HEXAGON_V6_vgathermhq_128B",
+    "llvm.hexagon.V6.vgathermhw" => "__builtin_HEXAGON_V6_vgathermhw",
+    "llvm.hexagon.V6.vgathermhw.128B" => "__builtin_HEXAGON_V6_vgathermhw_128B",
+    "llvm.hexagon.V6.vgathermhwq" => "__builtin_HEXAGON_V6_vgathermhwq",
+    "llvm.hexagon.V6.vgathermhwq.128B" => "__builtin_HEXAGON_V6_vgathermhwq_128B",
+    "llvm.hexagon.V6.vgathermw" => "__builtin_HEXAGON_V6_vgathermw",
+    "llvm.hexagon.V6.vgathermw.128B" => "__builtin_HEXAGON_V6_vgathermw_128B",
+    "llvm.hexagon.V6.vgathermwq" => "__builtin_HEXAGON_V6_vgathermwq",
+    "llvm.hexagon.V6.vgathermwq.128B" => "__builtin_HEXAGON_V6_vgathermwq_128B",
+    "llvm.hexagon.V6.vgtb" => "__builtin_HEXAGON_V6_vgtb",
+    "llvm.hexagon.V6.vgtb.128B" => "__builtin_HEXAGON_V6_vgtb_128B",
+    "llvm.hexagon.V6.vgtb.and" => "__builtin_HEXAGON_V6_vgtb_and",
+    "llvm.hexagon.V6.vgtb.and.128B" => "__builtin_HEXAGON_V6_vgtb_and_128B",
+    "llvm.hexagon.V6.vgtb.or" => "__builtin_HEXAGON_V6_vgtb_or",
+    "llvm.hexagon.V6.vgtb.or.128B" => "__builtin_HEXAGON_V6_vgtb_or_128B",
+    "llvm.hexagon.V6.vgtb.xor" => "__builtin_HEXAGON_V6_vgtb_xor",
+    "llvm.hexagon.V6.vgtb.xor.128B" => "__builtin_HEXAGON_V6_vgtb_xor_128B",
+    "llvm.hexagon.V6.vgtbf" => "__builtin_HEXAGON_V6_vgtbf",
+    "llvm.hexagon.V6.vgtbf.128B" => "__builtin_HEXAGON_V6_vgtbf_128B",
+    "llvm.hexagon.V6.vgtbf.and" => "__builtin_HEXAGON_V6_vgtbf_and",
+    "llvm.hexagon.V6.vgtbf.and.128B" => "__builtin_HEXAGON_V6_vgtbf_and_128B",
+    "llvm.hexagon.V6.vgtbf.or" => "__builtin_HEXAGON_V6_vgtbf_or",
+    "llvm.hexagon.V6.vgtbf.or.128B" => "__builtin_HEXAGON_V6_vgtbf_or_128B",
+    "llvm.hexagon.V6.vgtbf.xor" => "__builtin_HEXAGON_V6_vgtbf_xor",
+    "llvm.hexagon.V6.vgtbf.xor.128B" => "__builtin_HEXAGON_V6_vgtbf_xor_128B",
+    "llvm.hexagon.V6.vgth" => "__builtin_HEXAGON_V6_vgth",
+    "llvm.hexagon.V6.vgth.128B" => "__builtin_HEXAGON_V6_vgth_128B",
+    "llvm.hexagon.V6.vgth.and" => "__builtin_HEXAGON_V6_vgth_and",
+    "llvm.hexagon.V6.vgth.and.128B" => "__builtin_HEXAGON_V6_vgth_and_128B",
+    "llvm.hexagon.V6.vgth.or" => "__builtin_HEXAGON_V6_vgth_or",
+    "llvm.hexagon.V6.vgth.or.128B" => "__builtin_HEXAGON_V6_vgth_or_128B",
+    "llvm.hexagon.V6.vgth.xor" => "__builtin_HEXAGON_V6_vgth_xor",
+    "llvm.hexagon.V6.vgth.xor.128B" => "__builtin_HEXAGON_V6_vgth_xor_128B",
+    "llvm.hexagon.V6.vgthf" => "__builtin_HEXAGON_V6_vgthf",
+    "llvm.hexagon.V6.vgthf.128B" => "__builtin_HEXAGON_V6_vgthf_128B",
+    "llvm.hexagon.V6.vgthf.and" => "__builtin_HEXAGON_V6_vgthf_and",
+    "llvm.hexagon.V6.vgthf.and.128B" => "__builtin_HEXAGON_V6_vgthf_and_128B",
+    "llvm.hexagon.V6.vgthf.or" => "__builtin_HEXAGON_V6_vgthf_or",
+    "llvm.hexagon.V6.vgthf.or.128B" => "__builtin_HEXAGON_V6_vgthf_or_128B",
+    "llvm.hexagon.V6.vgthf.xor" => "__builtin_HEXAGON_V6_vgthf_xor",
+    "llvm.hexagon.V6.vgthf.xor.128B" => "__builtin_HEXAGON_V6_vgthf_xor_128B",
+    "llvm.hexagon.V6.vgtsf" => "__builtin_HEXAGON_V6_vgtsf",
+    "llvm.hexagon.V6.vgtsf.128B" => "__builtin_HEXAGON_V6_vgtsf_128B",
+    "llvm.hexagon.V6.vgtsf.and" => "__builtin_HEXAGON_V6_vgtsf_and",
+    "llvm.hexagon.V6.vgtsf.and.128B" => "__builtin_HEXAGON_V6_vgtsf_and_128B",
+    "llvm.hexagon.V6.vgtsf.or" => "__builtin_HEXAGON_V6_vgtsf_or",
+    "llvm.hexagon.V6.vgtsf.or.128B" => "__builtin_HEXAGON_V6_vgtsf_or_128B",
+    "llvm.hexagon.V6.vgtsf.xor" => "__builtin_HEXAGON_V6_vgtsf_xor",
+    "llvm.hexagon.V6.vgtsf.xor.128B" => "__builtin_HEXAGON_V6_vgtsf_xor_128B",
+    "llvm.hexagon.V6.vgtub" => "__builtin_HEXAGON_V6_vgtub",
+    "llvm.hexagon.V6.vgtub.128B" => "__builtin_HEXAGON_V6_vgtub_128B",
+    "llvm.hexagon.V6.vgtub.and" => "__builtin_HEXAGON_V6_vgtub_and",
+    "llvm.hexagon.V6.vgtub.and.128B" => "__builtin_HEXAGON_V6_vgtub_and_128B",
+    "llvm.hexagon.V6.vgtub.or" => "__builtin_HEXAGON_V6_vgtub_or",
+    "llvm.hexagon.V6.vgtub.or.128B" => "__builtin_HEXAGON_V6_vgtub_or_128B",
+    "llvm.hexagon.V6.vgtub.xor" => "__builtin_HEXAGON_V6_vgtub_xor",
+    "llvm.hexagon.V6.vgtub.xor.128B" => "__builtin_HEXAGON_V6_vgtub_xor_128B",
+    "llvm.hexagon.V6.vgtuh" => "__builtin_HEXAGON_V6_vgtuh",
+    "llvm.hexagon.V6.vgtuh.128B" => "__builtin_HEXAGON_V6_vgtuh_128B",
+    "llvm.hexagon.V6.vgtuh.and" => "__builtin_HEXAGON_V6_vgtuh_and",
+    "llvm.hexagon.V6.vgtuh.and.128B" => "__builtin_HEXAGON_V6_vgtuh_and_128B",
+    "llvm.hexagon.V6.vgtuh.or" => "__builtin_HEXAGON_V6_vgtuh_or",
+    "llvm.hexagon.V6.vgtuh.or.128B" => "__builtin_HEXAGON_V6_vgtuh_or_128B",
+    "llvm.hexagon.V6.vgtuh.xor" => "__builtin_HEXAGON_V6_vgtuh_xor",
+    "llvm.hexagon.V6.vgtuh.xor.128B" => "__builtin_HEXAGON_V6_vgtuh_xor_128B",
+    "llvm.hexagon.V6.vgtuw" => "__builtin_HEXAGON_V6_vgtuw",
+    "llvm.hexagon.V6.vgtuw.128B" => "__builtin_HEXAGON_V6_vgtuw_128B",
+    "llvm.hexagon.V6.vgtuw.and" => "__builtin_HEXAGON_V6_vgtuw_and",
+    "llvm.hexagon.V6.vgtuw.and.128B" => "__builtin_HEXAGON_V6_vgtuw_and_128B",
+    "llvm.hexagon.V6.vgtuw.or" => "__builtin_HEXAGON_V6_vgtuw_or",
+    "llvm.hexagon.V6.vgtuw.or.128B" => "__builtin_HEXAGON_V6_vgtuw_or_128B",
+    "llvm.hexagon.V6.vgtuw.xor" => "__builtin_HEXAGON_V6_vgtuw_xor",
+    "llvm.hexagon.V6.vgtuw.xor.128B" => "__builtin_HEXAGON_V6_vgtuw_xor_128B",
+    "llvm.hexagon.V6.vgtw" => "__builtin_HEXAGON_V6_vgtw",
+    "llvm.hexagon.V6.vgtw.128B" => "__builtin_HEXAGON_V6_vgtw_128B",
+    "llvm.hexagon.V6.vgtw.and" => "__builtin_HEXAGON_V6_vgtw_and",
+    "llvm.hexagon.V6.vgtw.and.128B" => "__builtin_HEXAGON_V6_vgtw_and_128B",
+    "llvm.hexagon.V6.vgtw.or" => "__builtin_HEXAGON_V6_vgtw_or",
+    "llvm.hexagon.V6.vgtw.or.128B" => "__builtin_HEXAGON_V6_vgtw_or_128B",
+    "llvm.hexagon.V6.vgtw.xor" => "__builtin_HEXAGON_V6_vgtw_xor",
+    "llvm.hexagon.V6.vgtw.xor.128B" => "__builtin_HEXAGON_V6_vgtw_xor_128B",
     "llvm.hexagon.V6.vinsertwr" => "__builtin_HEXAGON_V6_vinsertwr",
     "llvm.hexagon.V6.vinsertwr.128B" => "__builtin_HEXAGON_V6_vinsertwr_128B",
     "llvm.hexagon.V6.vlalignb" => "__builtin_HEXAGON_V6_vlalignb",
     "llvm.hexagon.V6.vlalignb.128B" => "__builtin_HEXAGON_V6_vlalignb_128B",
     "llvm.hexagon.V6.vlalignbi" => "__builtin_HEXAGON_V6_vlalignbi",
     "llvm.hexagon.V6.vlalignbi.128B" => "__builtin_HEXAGON_V6_vlalignbi_128B",
+    "llvm.hexagon.V6.vlsrb" => "__builtin_HEXAGON_V6_vlsrb",
+    "llvm.hexagon.V6.vlsrb.128B" => "__builtin_HEXAGON_V6_vlsrb_128B",
     "llvm.hexagon.V6.vlsrh" => "__builtin_HEXAGON_V6_vlsrh",
     "llvm.hexagon.V6.vlsrh.128B" => "__builtin_HEXAGON_V6_vlsrh_128B",
     "llvm.hexagon.V6.vlsrhv" => "__builtin_HEXAGON_V6_vlsrhv",
@@ -1310,6 +1743,8 @@ match name {
     "llvm.hexagon.V6.vlsrw.128B" => "__builtin_HEXAGON_V6_vlsrw_128B",
     "llvm.hexagon.V6.vlsrwv" => "__builtin_HEXAGON_V6_vlsrwv",
     "llvm.hexagon.V6.vlsrwv.128B" => "__builtin_HEXAGON_V6_vlsrwv_128B",
+    "llvm.hexagon.V6.vlut4" => "__builtin_HEXAGON_V6_vlut4",
+    "llvm.hexagon.V6.vlut4.128B" => "__builtin_HEXAGON_V6_vlut4_128B",
     "llvm.hexagon.V6.vlutb" => "__builtin_HEXAGON_V6_vlutb",
     "llvm.hexagon.V6.vlutb.128B" => "__builtin_HEXAGON_V6_vlutb_128B",
     "llvm.hexagon.V6.vlutb.acc" => "__builtin_HEXAGON_V6_vlutb_acc",
@@ -1320,12 +1755,32 @@ match name {
     "llvm.hexagon.V6.vlutb.dv.acc.128B" => "__builtin_HEXAGON_V6_vlutb_dv_acc_128B",
     "llvm.hexagon.V6.vlutvvb" => "__builtin_HEXAGON_V6_vlutvvb",
     "llvm.hexagon.V6.vlutvvb.128B" => "__builtin_HEXAGON_V6_vlutvvb_128B",
+    "llvm.hexagon.V6.vlutvvb.nm" => "__builtin_HEXAGON_V6_vlutvvb_nm",
+    "llvm.hexagon.V6.vlutvvb.nm.128B" => "__builtin_HEXAGON_V6_vlutvvb_nm_128B",
     "llvm.hexagon.V6.vlutvvb.oracc" => "__builtin_HEXAGON_V6_vlutvvb_oracc",
     "llvm.hexagon.V6.vlutvvb.oracc.128B" => "__builtin_HEXAGON_V6_vlutvvb_oracc_128B",
+    "llvm.hexagon.V6.vlutvvb.oracci" => "__builtin_HEXAGON_V6_vlutvvb_oracci",
+    "llvm.hexagon.V6.vlutvvb.oracci.128B" => "__builtin_HEXAGON_V6_vlutvvb_oracci_128B",
+    "llvm.hexagon.V6.vlutvvbi" => "__builtin_HEXAGON_V6_vlutvvbi",
+    "llvm.hexagon.V6.vlutvvbi.128B" => "__builtin_HEXAGON_V6_vlutvvbi_128B",
     "llvm.hexagon.V6.vlutvwh" => "__builtin_HEXAGON_V6_vlutvwh",
     "llvm.hexagon.V6.vlutvwh.128B" => "__builtin_HEXAGON_V6_vlutvwh_128B",
+    "llvm.hexagon.V6.vlutvwh.nm" => "__builtin_HEXAGON_V6_vlutvwh_nm",
+    "llvm.hexagon.V6.vlutvwh.nm.128B" => "__builtin_HEXAGON_V6_vlutvwh_nm_128B",
     "llvm.hexagon.V6.vlutvwh.oracc" => "__builtin_HEXAGON_V6_vlutvwh_oracc",
     "llvm.hexagon.V6.vlutvwh.oracc.128B" => "__builtin_HEXAGON_V6_vlutvwh_oracc_128B",
+    "llvm.hexagon.V6.vlutvwh.oracci" => "__builtin_HEXAGON_V6_vlutvwh_oracci",
+    "llvm.hexagon.V6.vlutvwh.oracci.128B" => "__builtin_HEXAGON_V6_vlutvwh_oracci_128B",
+    "llvm.hexagon.V6.vlutvwhi" => "__builtin_HEXAGON_V6_vlutvwhi",
+    "llvm.hexagon.V6.vlutvwhi.128B" => "__builtin_HEXAGON_V6_vlutvwhi_128B",
+    "llvm.hexagon.V6.vmax.bf" => "__builtin_HEXAGON_V6_vmax_bf",
+    "llvm.hexagon.V6.vmax.bf.128B" => "__builtin_HEXAGON_V6_vmax_bf_128B",
+    "llvm.hexagon.V6.vmax.hf" => "__builtin_HEXAGON_V6_vmax_hf",
+    "llvm.hexagon.V6.vmax.hf.128B" => "__builtin_HEXAGON_V6_vmax_hf_128B",
+    "llvm.hexagon.V6.vmax.sf" => "__builtin_HEXAGON_V6_vmax_sf",
+    "llvm.hexagon.V6.vmax.sf.128B" => "__builtin_HEXAGON_V6_vmax_sf_128B",
+    "llvm.hexagon.V6.vmaxb" => "__builtin_HEXAGON_V6_vmaxb",
+    "llvm.hexagon.V6.vmaxb.128B" => "__builtin_HEXAGON_V6_vmaxb_128B",
     "llvm.hexagon.V6.vmaxh" => "__builtin_HEXAGON_V6_vmaxh",
     "llvm.hexagon.V6.vmaxh.128B" => "__builtin_HEXAGON_V6_vmaxh_128B",
     "llvm.hexagon.V6.vmaxub" => "__builtin_HEXAGON_V6_vmaxub",
@@ -1334,6 +1789,14 @@ match name {
     "llvm.hexagon.V6.vmaxuh.128B" => "__builtin_HEXAGON_V6_vmaxuh_128B",
     "llvm.hexagon.V6.vmaxw" => "__builtin_HEXAGON_V6_vmaxw",
     "llvm.hexagon.V6.vmaxw.128B" => "__builtin_HEXAGON_V6_vmaxw_128B",
+    "llvm.hexagon.V6.vmin.bf" => "__builtin_HEXAGON_V6_vmin_bf",
+    "llvm.hexagon.V6.vmin.bf.128B" => "__builtin_HEXAGON_V6_vmin_bf_128B",
+    "llvm.hexagon.V6.vmin.hf" => "__builtin_HEXAGON_V6_vmin_hf",
+    "llvm.hexagon.V6.vmin.hf.128B" => "__builtin_HEXAGON_V6_vmin_hf_128B",
+    "llvm.hexagon.V6.vmin.sf" => "__builtin_HEXAGON_V6_vmin_sf",
+    "llvm.hexagon.V6.vmin.sf.128B" => "__builtin_HEXAGON_V6_vmin_sf_128B",
+    "llvm.hexagon.V6.vminb" => "__builtin_HEXAGON_V6_vminb",
+    "llvm.hexagon.V6.vminb.128B" => "__builtin_HEXAGON_V6_vminb_128B",
     "llvm.hexagon.V6.vminh" => "__builtin_HEXAGON_V6_vminh",
     "llvm.hexagon.V6.vminh.128B" => "__builtin_HEXAGON_V6_vminh_128B",
     "llvm.hexagon.V6.vminub" => "__builtin_HEXAGON_V6_vminub",
@@ -1348,12 +1811,56 @@ match name {
     "llvm.hexagon.V6.vmpabus.acc.128B" => "__builtin_HEXAGON_V6_vmpabus_acc_128B",
     "llvm.hexagon.V6.vmpabusv" => "__builtin_HEXAGON_V6_vmpabusv",
     "llvm.hexagon.V6.vmpabusv.128B" => "__builtin_HEXAGON_V6_vmpabusv_128B",
+    "llvm.hexagon.V6.vmpabuu" => "__builtin_HEXAGON_V6_vmpabuu",
+    "llvm.hexagon.V6.vmpabuu.128B" => "__builtin_HEXAGON_V6_vmpabuu_128B",
+    "llvm.hexagon.V6.vmpabuu.acc" => "__builtin_HEXAGON_V6_vmpabuu_acc",
+    "llvm.hexagon.V6.vmpabuu.acc.128B" => "__builtin_HEXAGON_V6_vmpabuu_acc_128B",
     "llvm.hexagon.V6.vmpabuuv" => "__builtin_HEXAGON_V6_vmpabuuv",
     "llvm.hexagon.V6.vmpabuuv.128B" => "__builtin_HEXAGON_V6_vmpabuuv_128B",
     "llvm.hexagon.V6.vmpahb" => "__builtin_HEXAGON_V6_vmpahb",
     "llvm.hexagon.V6.vmpahb.128B" => "__builtin_HEXAGON_V6_vmpahb_128B",
     "llvm.hexagon.V6.vmpahb.acc" => "__builtin_HEXAGON_V6_vmpahb_acc",
     "llvm.hexagon.V6.vmpahb.acc.128B" => "__builtin_HEXAGON_V6_vmpahb_acc_128B",
+    "llvm.hexagon.V6.vmpahhsat" => "__builtin_HEXAGON_V6_vmpahhsat",
+    "llvm.hexagon.V6.vmpahhsat.128B" => "__builtin_HEXAGON_V6_vmpahhsat_128B",
+    "llvm.hexagon.V6.vmpauhb" => "__builtin_HEXAGON_V6_vmpauhb",
+    "llvm.hexagon.V6.vmpauhb.128B" => "__builtin_HEXAGON_V6_vmpauhb_128B",
+    "llvm.hexagon.V6.vmpauhb.acc" => "__builtin_HEXAGON_V6_vmpauhb_acc",
+    "llvm.hexagon.V6.vmpauhb.acc.128B" => "__builtin_HEXAGON_V6_vmpauhb_acc_128B",
+    "llvm.hexagon.V6.vmpauhuhsat" => "__builtin_HEXAGON_V6_vmpauhuhsat",
+    "llvm.hexagon.V6.vmpauhuhsat.128B" => "__builtin_HEXAGON_V6_vmpauhuhsat_128B",
+    "llvm.hexagon.V6.vmpsuhuhsat" => "__builtin_HEXAGON_V6_vmpsuhuhsat",
+    "llvm.hexagon.V6.vmpsuhuhsat.128B" => "__builtin_HEXAGON_V6_vmpsuhuhsat_128B",
+    "llvm.hexagon.V6.vmpy.hf.hf" => "__builtin_HEXAGON_V6_vmpy_hf_hf",
+    "llvm.hexagon.V6.vmpy.hf.hf.128B" => "__builtin_HEXAGON_V6_vmpy_hf_hf_128B",
+    "llvm.hexagon.V6.vmpy.hf.hf.acc" => "__builtin_HEXAGON_V6_vmpy_hf_hf_acc",
+    "llvm.hexagon.V6.vmpy.hf.hf.acc.128B" => "__builtin_HEXAGON_V6_vmpy_hf_hf_acc_128B",
+    "llvm.hexagon.V6.vmpy.qf16" => "__builtin_HEXAGON_V6_vmpy_qf16",
+    "llvm.hexagon.V6.vmpy.qf16.128B" => "__builtin_HEXAGON_V6_vmpy_qf16_128B",
+    "llvm.hexagon.V6.vmpy.qf16.hf" => "__builtin_HEXAGON_V6_vmpy_qf16_hf",
+    "llvm.hexagon.V6.vmpy.qf16.hf.128B" => "__builtin_HEXAGON_V6_vmpy_qf16_hf_128B",
+    "llvm.hexagon.V6.vmpy.qf16.mix.hf" => "__builtin_HEXAGON_V6_vmpy_qf16_mix_hf",
+    "llvm.hexagon.V6.vmpy.qf16.mix.hf.128B" => "__builtin_HEXAGON_V6_vmpy_qf16_mix_hf_128B",
+    "llvm.hexagon.V6.vmpy.qf32" => "__builtin_HEXAGON_V6_vmpy_qf32",
+    "llvm.hexagon.V6.vmpy.qf32.128B" => "__builtin_HEXAGON_V6_vmpy_qf32_128B",
+    "llvm.hexagon.V6.vmpy.qf32.hf" => "__builtin_HEXAGON_V6_vmpy_qf32_hf",
+    "llvm.hexagon.V6.vmpy.qf32.hf.128B" => "__builtin_HEXAGON_V6_vmpy_qf32_hf_128B",
+    "llvm.hexagon.V6.vmpy.qf32.mix.hf" => "__builtin_HEXAGON_V6_vmpy_qf32_mix_hf",
+    "llvm.hexagon.V6.vmpy.qf32.mix.hf.128B" => "__builtin_HEXAGON_V6_vmpy_qf32_mix_hf_128B",
+    "llvm.hexagon.V6.vmpy.qf32.qf16" => "__builtin_HEXAGON_V6_vmpy_qf32_qf16",
+    "llvm.hexagon.V6.vmpy.qf32.qf16.128B" => "__builtin_HEXAGON_V6_vmpy_qf32_qf16_128B",
+    "llvm.hexagon.V6.vmpy.qf32.sf" => "__builtin_HEXAGON_V6_vmpy_qf32_sf",
+    "llvm.hexagon.V6.vmpy.qf32.sf.128B" => "__builtin_HEXAGON_V6_vmpy_qf32_sf_128B",
+    "llvm.hexagon.V6.vmpy.sf.bf" => "__builtin_HEXAGON_V6_vmpy_sf_bf",
+    "llvm.hexagon.V6.vmpy.sf.bf.128B" => "__builtin_HEXAGON_V6_vmpy_sf_bf_128B",
+    "llvm.hexagon.V6.vmpy.sf.bf.acc" => "__builtin_HEXAGON_V6_vmpy_sf_bf_acc",
+    "llvm.hexagon.V6.vmpy.sf.bf.acc.128B" => "__builtin_HEXAGON_V6_vmpy_sf_bf_acc_128B",
+    "llvm.hexagon.V6.vmpy.sf.hf" => "__builtin_HEXAGON_V6_vmpy_sf_hf",
+    "llvm.hexagon.V6.vmpy.sf.hf.128B" => "__builtin_HEXAGON_V6_vmpy_sf_hf_128B",
+    "llvm.hexagon.V6.vmpy.sf.hf.acc" => "__builtin_HEXAGON_V6_vmpy_sf_hf_acc",
+    "llvm.hexagon.V6.vmpy.sf.hf.acc.128B" => "__builtin_HEXAGON_V6_vmpy_sf_hf_acc_128B",
+    "llvm.hexagon.V6.vmpy.sf.sf" => "__builtin_HEXAGON_V6_vmpy_sf_sf",
+    "llvm.hexagon.V6.vmpy.sf.sf.128B" => "__builtin_HEXAGON_V6_vmpy_sf_sf_128B",
     "llvm.hexagon.V6.vmpybus" => "__builtin_HEXAGON_V6_vmpybus",
     "llvm.hexagon.V6.vmpybus.128B" => "__builtin_HEXAGON_V6_vmpybus_128B",
     "llvm.hexagon.V6.vmpybus.acc" => "__builtin_HEXAGON_V6_vmpybus_acc",
@@ -1368,8 +1875,12 @@ match name {
     "llvm.hexagon.V6.vmpybv.acc.128B" => "__builtin_HEXAGON_V6_vmpybv_acc_128B",
     "llvm.hexagon.V6.vmpyewuh" => "__builtin_HEXAGON_V6_vmpyewuh",
     "llvm.hexagon.V6.vmpyewuh.128B" => "__builtin_HEXAGON_V6_vmpyewuh_128B",
+    "llvm.hexagon.V6.vmpyewuh.64" => "__builtin_HEXAGON_V6_vmpyewuh_64",
+    "llvm.hexagon.V6.vmpyewuh.64.128B" => "__builtin_HEXAGON_V6_vmpyewuh_64_128B",
     "llvm.hexagon.V6.vmpyh" => "__builtin_HEXAGON_V6_vmpyh",
     "llvm.hexagon.V6.vmpyh.128B" => "__builtin_HEXAGON_V6_vmpyh_128B",
+    "llvm.hexagon.V6.vmpyh.acc" => "__builtin_HEXAGON_V6_vmpyh_acc",
+    "llvm.hexagon.V6.vmpyh.acc.128B" => "__builtin_HEXAGON_V6_vmpyh_acc_128B",
     "llvm.hexagon.V6.vmpyhsat.acc" => "__builtin_HEXAGON_V6_vmpyhsat_acc",
     "llvm.hexagon.V6.vmpyhsat.acc.128B" => "__builtin_HEXAGON_V6_vmpyhsat_acc_128B",
     "llvm.hexagon.V6.vmpyhsrs" => "__builtin_HEXAGON_V6_vmpyhsrs",
@@ -1412,8 +1923,14 @@ match name {
     "llvm.hexagon.V6.vmpyiwh.128B" => "__builtin_HEXAGON_V6_vmpyiwh_128B",
     "llvm.hexagon.V6.vmpyiwh.acc" => "__builtin_HEXAGON_V6_vmpyiwh_acc",
     "llvm.hexagon.V6.vmpyiwh.acc.128B" => "__builtin_HEXAGON_V6_vmpyiwh_acc_128B",
+    "llvm.hexagon.V6.vmpyiwub" => "__builtin_HEXAGON_V6_vmpyiwub",
+    "llvm.hexagon.V6.vmpyiwub.128B" => "__builtin_HEXAGON_V6_vmpyiwub_128B",
+    "llvm.hexagon.V6.vmpyiwub.acc" => "__builtin_HEXAGON_V6_vmpyiwub_acc",
+    "llvm.hexagon.V6.vmpyiwub.acc.128B" => "__builtin_HEXAGON_V6_vmpyiwub_acc_128B",
     "llvm.hexagon.V6.vmpyowh" => "__builtin_HEXAGON_V6_vmpyowh",
     "llvm.hexagon.V6.vmpyowh.128B" => "__builtin_HEXAGON_V6_vmpyowh_128B",
+    "llvm.hexagon.V6.vmpyowh.64.acc" => "__builtin_HEXAGON_V6_vmpyowh_64_acc",
+    "llvm.hexagon.V6.vmpyowh.64.acc.128B" => "__builtin_HEXAGON_V6_vmpyowh_64_acc_128B",
     "llvm.hexagon.V6.vmpyowh.rnd" => "__builtin_HEXAGON_V6_vmpyowh_rnd",
     "llvm.hexagon.V6.vmpyowh.rnd.128B" => "__builtin_HEXAGON_V6_vmpyowh_rnd_128B",
     "llvm.hexagon.V6.vmpyowh.rnd.sacc" => "__builtin_HEXAGON_V6_vmpyowh_rnd_sacc",
@@ -1432,10 +1949,20 @@ match name {
     "llvm.hexagon.V6.vmpyuh.128B" => "__builtin_HEXAGON_V6_vmpyuh_128B",
     "llvm.hexagon.V6.vmpyuh.acc" => "__builtin_HEXAGON_V6_vmpyuh_acc",
     "llvm.hexagon.V6.vmpyuh.acc.128B" => "__builtin_HEXAGON_V6_vmpyuh_acc_128B",
+    "llvm.hexagon.V6.vmpyuhe" => "__builtin_HEXAGON_V6_vmpyuhe",
+    "llvm.hexagon.V6.vmpyuhe.128B" => "__builtin_HEXAGON_V6_vmpyuhe_128B",
+    "llvm.hexagon.V6.vmpyuhe.acc" => "__builtin_HEXAGON_V6_vmpyuhe_acc",
+    "llvm.hexagon.V6.vmpyuhe.acc.128B" => "__builtin_HEXAGON_V6_vmpyuhe_acc_128B",
     "llvm.hexagon.V6.vmpyuhv" => "__builtin_HEXAGON_V6_vmpyuhv",
     "llvm.hexagon.V6.vmpyuhv.128B" => "__builtin_HEXAGON_V6_vmpyuhv_128B",
     "llvm.hexagon.V6.vmpyuhv.acc" => "__builtin_HEXAGON_V6_vmpyuhv_acc",
     "llvm.hexagon.V6.vmpyuhv.acc.128B" => "__builtin_HEXAGON_V6_vmpyuhv_acc_128B",
+    "llvm.hexagon.V6.vmpyuhvs" => "__builtin_HEXAGON_V6_vmpyuhvs",
+    "llvm.hexagon.V6.vmpyuhvs.128B" => "__builtin_HEXAGON_V6_vmpyuhvs_128B",
+    "llvm.hexagon.V6.vmux" => "__builtin_HEXAGON_V6_vmux",
+    "llvm.hexagon.V6.vmux.128B" => "__builtin_HEXAGON_V6_vmux_128B",
+    "llvm.hexagon.V6.vnavgb" => "__builtin_HEXAGON_V6_vnavgb",
+    "llvm.hexagon.V6.vnavgb.128B" => "__builtin_HEXAGON_V6_vnavgb_128B",
     "llvm.hexagon.V6.vnavgh" => "__builtin_HEXAGON_V6_vnavgh",
     "llvm.hexagon.V6.vnavgh.128B" => "__builtin_HEXAGON_V6_vnavgh_128B",
     "llvm.hexagon.V6.vnavgub" => "__builtin_HEXAGON_V6_vnavgub",
@@ -1468,8 +1995,18 @@ match name {
     "llvm.hexagon.V6.vpackwuh.sat.128B" => "__builtin_HEXAGON_V6_vpackwuh_sat_128B",
     "llvm.hexagon.V6.vpopcounth" => "__builtin_HEXAGON_V6_vpopcounth",
     "llvm.hexagon.V6.vpopcounth.128B" => "__builtin_HEXAGON_V6_vpopcounth_128B",
+    "llvm.hexagon.V6.vprefixqb" => "__builtin_HEXAGON_V6_vprefixqb",
+    "llvm.hexagon.V6.vprefixqb.128B" => "__builtin_HEXAGON_V6_vprefixqb_128B",
+    "llvm.hexagon.V6.vprefixqh" => "__builtin_HEXAGON_V6_vprefixqh",
+    "llvm.hexagon.V6.vprefixqh.128B" => "__builtin_HEXAGON_V6_vprefixqh_128B",
+    "llvm.hexagon.V6.vprefixqw" => "__builtin_HEXAGON_V6_vprefixqw",
+    "llvm.hexagon.V6.vprefixqw.128B" => "__builtin_HEXAGON_V6_vprefixqw_128B",
     "llvm.hexagon.V6.vrdelta" => "__builtin_HEXAGON_V6_vrdelta",
     "llvm.hexagon.V6.vrdelta.128B" => "__builtin_HEXAGON_V6_vrdelta_128B",
+    "llvm.hexagon.V6.vrmpybub.rtt" => "__builtin_HEXAGON_V6_vrmpybub_rtt",
+    "llvm.hexagon.V6.vrmpybub.rtt.128B" => "__builtin_HEXAGON_V6_vrmpybub_rtt_128B",
+    "llvm.hexagon.V6.vrmpybub.rtt.acc" => "__builtin_HEXAGON_V6_vrmpybub_rtt_acc",
+    "llvm.hexagon.V6.vrmpybub.rtt.acc.128B" => "__builtin_HEXAGON_V6_vrmpybub_rtt_acc_128B",
     "llvm.hexagon.V6.vrmpybus" => "__builtin_HEXAGON_V6_vrmpybus",
     "llvm.hexagon.V6.vrmpybus.128B" => "__builtin_HEXAGON_V6_vrmpybus_128B",
     "llvm.hexagon.V6.vrmpybus.acc" => "__builtin_HEXAGON_V6_vrmpybus_acc",
@@ -1490,6 +2027,10 @@ match name {
     "llvm.hexagon.V6.vrmpyub.128B" => "__builtin_HEXAGON_V6_vrmpyub_128B",
     "llvm.hexagon.V6.vrmpyub.acc" => "__builtin_HEXAGON_V6_vrmpyub_acc",
     "llvm.hexagon.V6.vrmpyub.acc.128B" => "__builtin_HEXAGON_V6_vrmpyub_acc_128B",
+    "llvm.hexagon.V6.vrmpyub.rtt" => "__builtin_HEXAGON_V6_vrmpyub_rtt",
+    "llvm.hexagon.V6.vrmpyub.rtt.128B" => "__builtin_HEXAGON_V6_vrmpyub_rtt_128B",
+    "llvm.hexagon.V6.vrmpyub.rtt.acc" => "__builtin_HEXAGON_V6_vrmpyub_rtt_acc",
+    "llvm.hexagon.V6.vrmpyub.rtt.acc.128B" => "__builtin_HEXAGON_V6_vrmpyub_rtt_acc_128B",
     "llvm.hexagon.V6.vrmpyubi" => "__builtin_HEXAGON_V6_vrmpyubi",
     "llvm.hexagon.V6.vrmpyubi.128B" => "__builtin_HEXAGON_V6_vrmpyubi_128B",
     "llvm.hexagon.V6.vrmpyubi.acc" => "__builtin_HEXAGON_V6_vrmpyubi_acc",
@@ -1500,10 +2041,16 @@ match name {
     "llvm.hexagon.V6.vrmpyubv.acc.128B" => "__builtin_HEXAGON_V6_vrmpyubv_acc_128B",
     "llvm.hexagon.V6.vror" => "__builtin_HEXAGON_V6_vror",
     "llvm.hexagon.V6.vror.128B" => "__builtin_HEXAGON_V6_vror_128B",
+    "llvm.hexagon.V6.vrotr" => "__builtin_HEXAGON_V6_vrotr",
+    "llvm.hexagon.V6.vrotr.128B" => "__builtin_HEXAGON_V6_vrotr_128B",
     "llvm.hexagon.V6.vroundhb" => "__builtin_HEXAGON_V6_vroundhb",
     "llvm.hexagon.V6.vroundhb.128B" => "__builtin_HEXAGON_V6_vroundhb_128B",
     "llvm.hexagon.V6.vroundhub" => "__builtin_HEXAGON_V6_vroundhub",
     "llvm.hexagon.V6.vroundhub.128B" => "__builtin_HEXAGON_V6_vroundhub_128B",
+    "llvm.hexagon.V6.vrounduhub" => "__builtin_HEXAGON_V6_vrounduhub",
+    "llvm.hexagon.V6.vrounduhub.128B" => "__builtin_HEXAGON_V6_vrounduhub_128B",
+    "llvm.hexagon.V6.vrounduwuh" => "__builtin_HEXAGON_V6_vrounduwuh",
+    "llvm.hexagon.V6.vrounduwuh.128B" => "__builtin_HEXAGON_V6_vrounduwuh_128B",
     "llvm.hexagon.V6.vroundwh" => "__builtin_HEXAGON_V6_vroundwh",
     "llvm.hexagon.V6.vroundwh.128B" => "__builtin_HEXAGON_V6_vroundwh_128B",
     "llvm.hexagon.V6.vroundwuh" => "__builtin_HEXAGON_V6_vroundwuh",
@@ -1512,12 +2059,34 @@ match name {
     "llvm.hexagon.V6.vrsadubi.128B" => "__builtin_HEXAGON_V6_vrsadubi_128B",
     "llvm.hexagon.V6.vrsadubi.acc" => "__builtin_HEXAGON_V6_vrsadubi_acc",
     "llvm.hexagon.V6.vrsadubi.acc.128B" => "__builtin_HEXAGON_V6_vrsadubi_acc_128B",
+    "llvm.hexagon.V6.vsatdw" => "__builtin_HEXAGON_V6_vsatdw",
+    "llvm.hexagon.V6.vsatdw.128B" => "__builtin_HEXAGON_V6_vsatdw_128B",
     "llvm.hexagon.V6.vsathub" => "__builtin_HEXAGON_V6_vsathub",
     "llvm.hexagon.V6.vsathub.128B" => "__builtin_HEXAGON_V6_vsathub_128B",
+    "llvm.hexagon.V6.vsatuwuh" => "__builtin_HEXAGON_V6_vsatuwuh",
+    "llvm.hexagon.V6.vsatuwuh.128B" => "__builtin_HEXAGON_V6_vsatuwuh_128B",
     "llvm.hexagon.V6.vsatwh" => "__builtin_HEXAGON_V6_vsatwh",
     "llvm.hexagon.V6.vsatwh.128B" => "__builtin_HEXAGON_V6_vsatwh_128B",
     "llvm.hexagon.V6.vsb" => "__builtin_HEXAGON_V6_vsb",
     "llvm.hexagon.V6.vsb.128B" => "__builtin_HEXAGON_V6_vsb_128B",
+    "llvm.hexagon.V6.vscattermh" => "__builtin_HEXAGON_V6_vscattermh",
+    "llvm.hexagon.V6.vscattermh.128B" => "__builtin_HEXAGON_V6_vscattermh_128B",
+    "llvm.hexagon.V6.vscattermh.add" => "__builtin_HEXAGON_V6_vscattermh_add",
+    "llvm.hexagon.V6.vscattermh.add.128B" => "__builtin_HEXAGON_V6_vscattermh_add_128B",
+    "llvm.hexagon.V6.vscattermhq" => "__builtin_HEXAGON_V6_vscattermhq",
+    "llvm.hexagon.V6.vscattermhq.128B" => "__builtin_HEXAGON_V6_vscattermhq_128B",
+    "llvm.hexagon.V6.vscattermhw" => "__builtin_HEXAGON_V6_vscattermhw",
+    "llvm.hexagon.V6.vscattermhw.128B" => "__builtin_HEXAGON_V6_vscattermhw_128B",
+    "llvm.hexagon.V6.vscattermhw.add" => "__builtin_HEXAGON_V6_vscattermhw_add",
+    "llvm.hexagon.V6.vscattermhw.add.128B" => "__builtin_HEXAGON_V6_vscattermhw_add_128B",
+    "llvm.hexagon.V6.vscattermhwq" => "__builtin_HEXAGON_V6_vscattermhwq",
+    "llvm.hexagon.V6.vscattermhwq.128B" => "__builtin_HEXAGON_V6_vscattermhwq_128B",
+    "llvm.hexagon.V6.vscattermw" => "__builtin_HEXAGON_V6_vscattermw",
+    "llvm.hexagon.V6.vscattermw.128B" => "__builtin_HEXAGON_V6_vscattermw_128B",
+    "llvm.hexagon.V6.vscattermw.add" => "__builtin_HEXAGON_V6_vscattermw_add",
+    "llvm.hexagon.V6.vscattermw.add.128B" => "__builtin_HEXAGON_V6_vscattermw_add_128B",
+    "llvm.hexagon.V6.vscattermwq" => "__builtin_HEXAGON_V6_vscattermwq",
+    "llvm.hexagon.V6.vscattermwq.128B" => "__builtin_HEXAGON_V6_vscattermwq_128B",
     "llvm.hexagon.V6.vsh" => "__builtin_HEXAGON_V6_vsh",
     "llvm.hexagon.V6.vsh.128B" => "__builtin_HEXAGON_V6_vsh_128B",
     "llvm.hexagon.V6.vshufeh" => "__builtin_HEXAGON_V6_vshufeh",
@@ -1538,14 +2107,46 @@ match name {
     "llvm.hexagon.V6.vshufoeh.128B" => "__builtin_HEXAGON_V6_vshufoeh_128B",
     "llvm.hexagon.V6.vshufoh" => "__builtin_HEXAGON_V6_vshufoh",
     "llvm.hexagon.V6.vshufoh.128B" => "__builtin_HEXAGON_V6_vshufoh_128B",
+    "llvm.hexagon.V6.vsub.hf" => "__builtin_HEXAGON_V6_vsub_hf",
+    "llvm.hexagon.V6.vsub.hf.128B" => "__builtin_HEXAGON_V6_vsub_hf_128B",
+    "llvm.hexagon.V6.vsub.hf.hf" => "__builtin_HEXAGON_V6_vsub_hf_hf",
+    "llvm.hexagon.V6.vsub.hf.hf.128B" => "__builtin_HEXAGON_V6_vsub_hf_hf_128B",
+    "llvm.hexagon.V6.vsub.qf16" => "__builtin_HEXAGON_V6_vsub_qf16",
+    "llvm.hexagon.V6.vsub.qf16.128B" => "__builtin_HEXAGON_V6_vsub_qf16_128B",
+    "llvm.hexagon.V6.vsub.qf16.mix" => "__builtin_HEXAGON_V6_vsub_qf16_mix",
+    "llvm.hexagon.V6.vsub.qf16.mix.128B" => "__builtin_HEXAGON_V6_vsub_qf16_mix_128B",
+    "llvm.hexagon.V6.vsub.qf32" => "__builtin_HEXAGON_V6_vsub_qf32",
+    "llvm.hexagon.V6.vsub.qf32.128B" => "__builtin_HEXAGON_V6_vsub_qf32_128B",
+    "llvm.hexagon.V6.vsub.qf32.mix" => "__builtin_HEXAGON_V6_vsub_qf32_mix",
+    "llvm.hexagon.V6.vsub.qf32.mix.128B" => "__builtin_HEXAGON_V6_vsub_qf32_mix_128B",
+    "llvm.hexagon.V6.vsub.sf" => "__builtin_HEXAGON_V6_vsub_sf",
+    "llvm.hexagon.V6.vsub.sf.128B" => "__builtin_HEXAGON_V6_vsub_sf_128B",
+    "llvm.hexagon.V6.vsub.sf.bf" => "__builtin_HEXAGON_V6_vsub_sf_bf",
+    "llvm.hexagon.V6.vsub.sf.bf.128B" => "__builtin_HEXAGON_V6_vsub_sf_bf_128B",
+    "llvm.hexagon.V6.vsub.sf.hf" => "__builtin_HEXAGON_V6_vsub_sf_hf",
+    "llvm.hexagon.V6.vsub.sf.hf.128B" => "__builtin_HEXAGON_V6_vsub_sf_hf_128B",
+    "llvm.hexagon.V6.vsub.sf.sf" => "__builtin_HEXAGON_V6_vsub_sf_sf",
+    "llvm.hexagon.V6.vsub.sf.sf.128B" => "__builtin_HEXAGON_V6_vsub_sf_sf_128B",
     "llvm.hexagon.V6.vsubb" => "__builtin_HEXAGON_V6_vsubb",
     "llvm.hexagon.V6.vsubb.128B" => "__builtin_HEXAGON_V6_vsubb_128B",
     "llvm.hexagon.V6.vsubb.dv" => "__builtin_HEXAGON_V6_vsubb_dv",
     "llvm.hexagon.V6.vsubb.dv.128B" => "__builtin_HEXAGON_V6_vsubb_dv_128B",
+    "llvm.hexagon.V6.vsubbnq" => "__builtin_HEXAGON_V6_vsubbnq",
+    "llvm.hexagon.V6.vsubbnq.128B" => "__builtin_HEXAGON_V6_vsubbnq_128B",
+    "llvm.hexagon.V6.vsubbq" => "__builtin_HEXAGON_V6_vsubbq",
+    "llvm.hexagon.V6.vsubbq.128B" => "__builtin_HEXAGON_V6_vsubbq_128B",
+    "llvm.hexagon.V6.vsubbsat" => "__builtin_HEXAGON_V6_vsubbsat",
+    "llvm.hexagon.V6.vsubbsat.128B" => "__builtin_HEXAGON_V6_vsubbsat_128B",
+    "llvm.hexagon.V6.vsubbsat.dv" => "__builtin_HEXAGON_V6_vsubbsat_dv",
+    "llvm.hexagon.V6.vsubbsat.dv.128B" => "__builtin_HEXAGON_V6_vsubbsat_dv_128B",
     "llvm.hexagon.V6.vsubh" => "__builtin_HEXAGON_V6_vsubh",
     "llvm.hexagon.V6.vsubh.128B" => "__builtin_HEXAGON_V6_vsubh_128B",
     "llvm.hexagon.V6.vsubh.dv" => "__builtin_HEXAGON_V6_vsubh_dv",
     "llvm.hexagon.V6.vsubh.dv.128B" => "__builtin_HEXAGON_V6_vsubh_dv_128B",
+    "llvm.hexagon.V6.vsubhnq" => "__builtin_HEXAGON_V6_vsubhnq",
+    "llvm.hexagon.V6.vsubhnq.128B" => "__builtin_HEXAGON_V6_vsubhnq_128B",
+    "llvm.hexagon.V6.vsubhq" => "__builtin_HEXAGON_V6_vsubhq",
+    "llvm.hexagon.V6.vsubhq.128B" => "__builtin_HEXAGON_V6_vsubhq_128B",
     "llvm.hexagon.V6.vsubhsat" => "__builtin_HEXAGON_V6_vsubhsat",
     "llvm.hexagon.V6.vsubhsat.128B" => "__builtin_HEXAGON_V6_vsubhsat_128B",
     "llvm.hexagon.V6.vsubhsat.dv" => "__builtin_HEXAGON_V6_vsubhsat_dv",
@@ -1558,20 +2159,32 @@ match name {
     "llvm.hexagon.V6.vsububsat.128B" => "__builtin_HEXAGON_V6_vsububsat_128B",
     "llvm.hexagon.V6.vsububsat.dv" => "__builtin_HEXAGON_V6_vsububsat_dv",
     "llvm.hexagon.V6.vsububsat.dv.128B" => "__builtin_HEXAGON_V6_vsububsat_dv_128B",
+    "llvm.hexagon.V6.vsubububb.sat" => "__builtin_HEXAGON_V6_vsubububb_sat",
+    "llvm.hexagon.V6.vsubububb.sat.128B" => "__builtin_HEXAGON_V6_vsubububb_sat_128B",
     "llvm.hexagon.V6.vsubuhsat" => "__builtin_HEXAGON_V6_vsubuhsat",
     "llvm.hexagon.V6.vsubuhsat.128B" => "__builtin_HEXAGON_V6_vsubuhsat_128B",
     "llvm.hexagon.V6.vsubuhsat.dv" => "__builtin_HEXAGON_V6_vsubuhsat_dv",
     "llvm.hexagon.V6.vsubuhsat.dv.128B" => "__builtin_HEXAGON_V6_vsubuhsat_dv_128B",
     "llvm.hexagon.V6.vsubuhw" => "__builtin_HEXAGON_V6_vsubuhw",
     "llvm.hexagon.V6.vsubuhw.128B" => "__builtin_HEXAGON_V6_vsubuhw_128B",
+    "llvm.hexagon.V6.vsubuwsat" => "__builtin_HEXAGON_V6_vsubuwsat",
+    "llvm.hexagon.V6.vsubuwsat.128B" => "__builtin_HEXAGON_V6_vsubuwsat_128B",
+    "llvm.hexagon.V6.vsubuwsat.dv" => "__builtin_HEXAGON_V6_vsubuwsat_dv",
+    "llvm.hexagon.V6.vsubuwsat.dv.128B" => "__builtin_HEXAGON_V6_vsubuwsat_dv_128B",
     "llvm.hexagon.V6.vsubw" => "__builtin_HEXAGON_V6_vsubw",
     "llvm.hexagon.V6.vsubw.128B" => "__builtin_HEXAGON_V6_vsubw_128B",
     "llvm.hexagon.V6.vsubw.dv" => "__builtin_HEXAGON_V6_vsubw_dv",
     "llvm.hexagon.V6.vsubw.dv.128B" => "__builtin_HEXAGON_V6_vsubw_dv_128B",
+    "llvm.hexagon.V6.vsubwnq" => "__builtin_HEXAGON_V6_vsubwnq",
+    "llvm.hexagon.V6.vsubwnq.128B" => "__builtin_HEXAGON_V6_vsubwnq_128B",
+    "llvm.hexagon.V6.vsubwq" => "__builtin_HEXAGON_V6_vsubwq",
+    "llvm.hexagon.V6.vsubwq.128B" => "__builtin_HEXAGON_V6_vsubwq_128B",
     "llvm.hexagon.V6.vsubwsat" => "__builtin_HEXAGON_V6_vsubwsat",
     "llvm.hexagon.V6.vsubwsat.128B" => "__builtin_HEXAGON_V6_vsubwsat_128B",
     "llvm.hexagon.V6.vsubwsat.dv" => "__builtin_HEXAGON_V6_vsubwsat_dv",
     "llvm.hexagon.V6.vsubwsat.dv.128B" => "__builtin_HEXAGON_V6_vsubwsat_dv_128B",
+    "llvm.hexagon.V6.vswap" => "__builtin_HEXAGON_V6_vswap",
+    "llvm.hexagon.V6.vswap.128B" => "__builtin_HEXAGON_V6_vswap_128B",
     "llvm.hexagon.V6.vtmpyb" => "__builtin_HEXAGON_V6_vtmpyb",
     "llvm.hexagon.V6.vtmpyb.128B" => "__builtin_HEXAGON_V6_vtmpyb_128B",
     "llvm.hexagon.V6.vtmpyb.acc" => "__builtin_HEXAGON_V6_vtmpyb_acc",
@@ -1602,6 +2215,19 @@ match name {
     "llvm.hexagon.V6.vzb.128B" => "__builtin_HEXAGON_V6_vzb_128B",
     "llvm.hexagon.V6.vzh" => "__builtin_HEXAGON_V6_vzh",
     "llvm.hexagon.V6.vzh.128B" => "__builtin_HEXAGON_V6_vzh_128B",
+    "llvm.hexagon.Y2.dccleana" => "__builtin_HEXAGON_Y2_dccleana",
+    "llvm.hexagon.Y2.dccleaninva" => "__builtin_HEXAGON_Y2_dccleaninva",
+    "llvm.hexagon.Y2.dcfetch" => "__builtin_HEXAGON_Y2_dcfetch",
+    "llvm.hexagon.Y2.dcinva" => "__builtin_HEXAGON_Y2_dcinva",
+    "llvm.hexagon.Y2.dczeroa" => "__builtin_HEXAGON_Y2_dczeroa",
+    "llvm.hexagon.Y4.l2fetch" => "__builtin_HEXAGON_Y4_l2fetch",
+    "llvm.hexagon.Y5.l2fetch" => "__builtin_HEXAGON_Y5_l2fetch",
+    "llvm.hexagon.Y6.dmlink" => "__builtin_HEXAGON_Y6_dmlink",
+    "llvm.hexagon.Y6.dmpause" => "__builtin_HEXAGON_Y6_dmpause",
+    "llvm.hexagon.Y6.dmpoll" => "__builtin_HEXAGON_Y6_dmpoll",
+    "llvm.hexagon.Y6.dmresume" => "__builtin_HEXAGON_Y6_dmresume",
+    "llvm.hexagon.Y6.dmstart" => "__builtin_HEXAGON_Y6_dmstart",
+    "llvm.hexagon.Y6.dmwait" => "__builtin_HEXAGON_Y6_dmwait",
     "llvm.hexagon.brev.ldb" => "__builtin_brev_ldb",
     "llvm.hexagon.brev.ldd" => "__builtin_brev_ldd",
     "llvm.hexagon.brev.ldh" => "__builtin_brev_ldh",
@@ -1626,6 +2252,8 @@ match name {
     "llvm.hexagon.circ.stw" => "__builtin_circ_stw",
     "llvm.hexagon.mm256i.vaddw" => "__builtin__mm256i_vaddw",
     "llvm.hexagon.prefetch" => "__builtin_HEXAGON_prefetch",
+    "llvm.hexagon.vmemcpy" => "__builtin_hexagon_vmemcpy",
+    "llvm.hexagon.vmemset" => "__builtin_hexagon_vmemset",
     // mips
     "llvm.mips.absq.s.ph" => "__builtin_mips_absq_s_ph",
     "llvm.mips.absq.s.qb" => "__builtin_mips_absq_s_qb",
@@ -2299,6 +2927,8 @@ match name {
     "llvm.mips.xor.v" => "__builtin_msa_xor_v",
     "llvm.mips.xori.b" => "__builtin_msa_xori_b",
     // nvvm
+    "llvm.nvvm.abs.bf16" => "__nvvm_abs_bf16",
+    "llvm.nvvm.abs.bf16x2" => "__nvvm_abs_bf16x2",
     "llvm.nvvm.abs.i" => "__nvvm_abs_i",
     "llvm.nvvm.abs.ll" => "__nvvm_abs_ll",
     "llvm.nvvm.add.rm.d" => "__nvvm_add_rm_d",
@@ -2314,8 +2944,13 @@ match name {
     "llvm.nvvm.add.rz.f" => "__nvvm_add_rz_f",
     "llvm.nvvm.add.rz.ftz.f" => "__nvvm_add_rz_ftz_f",
     "llvm.nvvm.bar.sync" => "__nvvm_bar_sync",
-    "llvm.nvvm.barrier0" => "__nvvm_bar0",
-    // [DUPLICATE]: "llvm.nvvm.barrier0" => "__syncthreads",
+    "llvm.nvvm.bar.warp.sync" => "__nvvm_bar_warp_sync",
+    "llvm.nvvm.barrier" => "__nvvm_bar",
+    "llvm.nvvm.barrier.n" => "__nvvm_bar_n",
+    "llvm.nvvm.barrier.sync" => "__nvvm_barrier_sync",
+    "llvm.nvvm.barrier.sync.cnt" => "__nvvm_barrier_sync_cnt",
+    "llvm.nvvm.barrier0" => "__syncthreads",
+    // [DUPLICATE]: "llvm.nvvm.barrier0" => "__nvvm_bar0",
     "llvm.nvvm.barrier0.and" => "__nvvm_bar0_and",
     "llvm.nvvm.barrier0.or" => "__nvvm_bar0_or",
     "llvm.nvvm.barrier0.popc" => "__nvvm_bar0_popc",
@@ -2332,6 +2967,17 @@ match name {
     "llvm.nvvm.clz.ll" => "__nvvm_clz_ll",
     "llvm.nvvm.cos.approx.f" => "__nvvm_cos_approx_f",
     "llvm.nvvm.cos.approx.ftz.f" => "__nvvm_cos_approx_ftz_f",
+    "llvm.nvvm.cp.async.ca.shared.global.16" => "__nvvm_cp_async_ca_shared_global_16",
+    "llvm.nvvm.cp.async.ca.shared.global.4" => "__nvvm_cp_async_ca_shared_global_4",
+    "llvm.nvvm.cp.async.ca.shared.global.8" => "__nvvm_cp_async_ca_shared_global_8",
+    "llvm.nvvm.cp.async.cg.shared.global.16" => "__nvvm_cp_async_cg_shared_global_16",
+    "llvm.nvvm.cp.async.commit.group" => "__nvvm_cp_async_commit_group",
+    "llvm.nvvm.cp.async.mbarrier.arrive" => "__nvvm_cp_async_mbarrier_arrive",
+    "llvm.nvvm.cp.async.mbarrier.arrive.noinc" => "__nvvm_cp_async_mbarrier_arrive_noinc",
+    "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" => "__nvvm_cp_async_mbarrier_arrive_noinc_shared",
+    "llvm.nvvm.cp.async.mbarrier.arrive.shared" => "__nvvm_cp_async_mbarrier_arrive_shared",
+    "llvm.nvvm.cp.async.wait.all" => "__nvvm_cp_async_wait_all",
+    "llvm.nvvm.cp.async.wait.group" => "__nvvm_cp_async_wait_group",
     "llvm.nvvm.d2f.rm" => "__nvvm_d2f_rm",
     "llvm.nvvm.d2f.rm.ftz" => "__nvvm_d2f_rm_ftz",
     "llvm.nvvm.d2f.rn" => "__nvvm_d2f_rn",
@@ -2374,7 +3020,13 @@ match name {
     "llvm.nvvm.div.rz.ftz.f" => "__nvvm_div_rz_ftz_f",
     "llvm.nvvm.ex2.approx.d" => "__nvvm_ex2_approx_d",
     "llvm.nvvm.ex2.approx.f" => "__nvvm_ex2_approx_f",
+    "llvm.nvvm.ex2.approx.f16" => "__nvvm_ex2_approx_f16",
+    "llvm.nvvm.ex2.approx.f16x2" => "__nvvm_ex2_approx_f16x2",
     "llvm.nvvm.ex2.approx.ftz.f" => "__nvvm_ex2_approx_ftz_f",
+    "llvm.nvvm.f2bf16.rn" => "__nvvm_f2bf16_rn",
+    "llvm.nvvm.f2bf16.rn.relu" => "__nvvm_f2bf16_rn_relu",
+    "llvm.nvvm.f2bf16.rz" => "__nvvm_f2bf16_rz",
+    "llvm.nvvm.f2bf16.rz.relu" => "__nvvm_f2bf16_rz_relu",
     "llvm.nvvm.f2h.rn" => "__nvvm_f2h_rn",
     "llvm.nvvm.f2h.rn.ftz" => "__nvvm_f2h_rn_ftz",
     "llvm.nvvm.f2i.rm" => "__nvvm_f2i_rm",
@@ -2393,6 +3045,7 @@ match name {
     "llvm.nvvm.f2ll.rp.ftz" => "__nvvm_f2ll_rp_ftz",
     "llvm.nvvm.f2ll.rz" => "__nvvm_f2ll_rz",
     "llvm.nvvm.f2ll.rz.ftz" => "__nvvm_f2ll_rz_ftz",
+    "llvm.nvvm.f2tf32.rna" => "__nvvm_f2tf32_rna",
     "llvm.nvvm.f2ui.rm" => "__nvvm_f2ui_rm",
     "llvm.nvvm.f2ui.rm.ftz" => "__nvvm_f2ui_rm_ftz",
     "llvm.nvvm.f2ui.rn" => "__nvvm_f2ui_rn",
@@ -2412,27 +3065,112 @@ match name {
     "llvm.nvvm.fabs.d" => "__nvvm_fabs_d",
     "llvm.nvvm.fabs.f" => "__nvvm_fabs_f",
     "llvm.nvvm.fabs.ftz.f" => "__nvvm_fabs_ftz_f",
+    "llvm.nvvm.ff2bf16x2.rn" => "__nvvm_ff2bf16x2_rn",
+    "llvm.nvvm.ff2bf16x2.rn.relu" => "__nvvm_ff2bf16x2_rn_relu",
+    "llvm.nvvm.ff2bf16x2.rz" => "__nvvm_ff2bf16x2_rz",
+    "llvm.nvvm.ff2bf16x2.rz.relu" => "__nvvm_ff2bf16x2_rz_relu",
+    "llvm.nvvm.ff2f16x2.rn" => "__nvvm_ff2f16x2_rn",
+    "llvm.nvvm.ff2f16x2.rn.relu" => "__nvvm_ff2f16x2_rn_relu",
+    "llvm.nvvm.ff2f16x2.rz" => "__nvvm_ff2f16x2_rz",
+    "llvm.nvvm.ff2f16x2.rz.relu" => "__nvvm_ff2f16x2_rz_relu",
     "llvm.nvvm.floor.d" => "__nvvm_floor_d",
     "llvm.nvvm.floor.f" => "__nvvm_floor_f",
     "llvm.nvvm.floor.ftz.f" => "__nvvm_floor_ftz_f",
     "llvm.nvvm.fma.rm.d" => "__nvvm_fma_rm_d",
     "llvm.nvvm.fma.rm.f" => "__nvvm_fma_rm_f",
     "llvm.nvvm.fma.rm.ftz.f" => "__nvvm_fma_rm_ftz_f",
+    "llvm.nvvm.fma.rn.bf16" => "__nvvm_fma_rn_bf16",
+    "llvm.nvvm.fma.rn.bf16x2" => "__nvvm_fma_rn_bf16x2",
     "llvm.nvvm.fma.rn.d" => "__nvvm_fma_rn_d",
     "llvm.nvvm.fma.rn.f" => "__nvvm_fma_rn_f",
+    "llvm.nvvm.fma.rn.f16" => "__nvvm_fma_rn_f16",
+    "llvm.nvvm.fma.rn.f16x2" => "__nvvm_fma_rn_f16x2",
     "llvm.nvvm.fma.rn.ftz.f" => "__nvvm_fma_rn_ftz_f",
+    "llvm.nvvm.fma.rn.ftz.f16" => "__nvvm_fma_rn_ftz_f16",
+    "llvm.nvvm.fma.rn.ftz.f16x2" => "__nvvm_fma_rn_ftz_f16x2",
+    "llvm.nvvm.fma.rn.ftz.relu.f16" => "__nvvm_fma_rn_ftz_relu_f16",
+    "llvm.nvvm.fma.rn.ftz.relu.f16x2" => "__nvvm_fma_rn_ftz_relu_f16x2",
+    "llvm.nvvm.fma.rn.ftz.sat.f16" => "__nvvm_fma_rn_ftz_sat_f16",
+    "llvm.nvvm.fma.rn.ftz.sat.f16x2" => "__nvvm_fma_rn_ftz_sat_f16x2",
+    "llvm.nvvm.fma.rn.relu.bf16" => "__nvvm_fma_rn_relu_bf16",
+    "llvm.nvvm.fma.rn.relu.bf16x2" => "__nvvm_fma_rn_relu_bf16x2",
+    "llvm.nvvm.fma.rn.relu.f16" => "__nvvm_fma_rn_relu_f16",
+    "llvm.nvvm.fma.rn.relu.f16x2" => "__nvvm_fma_rn_relu_f16x2",
+    "llvm.nvvm.fma.rn.sat.f16" => "__nvvm_fma_rn_sat_f16",
+    "llvm.nvvm.fma.rn.sat.f16x2" => "__nvvm_fma_rn_sat_f16x2",
     "llvm.nvvm.fma.rp.d" => "__nvvm_fma_rp_d",
     "llvm.nvvm.fma.rp.f" => "__nvvm_fma_rp_f",
     "llvm.nvvm.fma.rp.ftz.f" => "__nvvm_fma_rp_ftz_f",
     "llvm.nvvm.fma.rz.d" => "__nvvm_fma_rz_d",
     "llvm.nvvm.fma.rz.f" => "__nvvm_fma_rz_f",
     "llvm.nvvm.fma.rz.ftz.f" => "__nvvm_fma_rz_ftz_f",
+    "llvm.nvvm.fmax.bf16" => "__nvvm_fmax_bf16",
+    "llvm.nvvm.fmax.bf16x2" => "__nvvm_fmax_bf16x2",
     "llvm.nvvm.fmax.d" => "__nvvm_fmax_d",
     "llvm.nvvm.fmax.f" => "__nvvm_fmax_f",
+    "llvm.nvvm.fmax.f16" => "__nvvm_fmax_f16",
+    "llvm.nvvm.fmax.f16x2" => "__nvvm_fmax_f16x2",
     "llvm.nvvm.fmax.ftz.f" => "__nvvm_fmax_ftz_f",
+    "llvm.nvvm.fmax.ftz.f16" => "__nvvm_fmax_ftz_f16",
+    "llvm.nvvm.fmax.ftz.f16x2" => "__nvvm_fmax_ftz_f16x2",
+    "llvm.nvvm.fmax.ftz.nan.f" => "__nvvm_fmax_ftz_nan_f",
+    "llvm.nvvm.fmax.ftz.nan.f16" => "__nvvm_fmax_ftz_nan_f16",
+    "llvm.nvvm.fmax.ftz.nan.f16x2" => "__nvvm_fmax_ftz_nan_f16x2",
+    "llvm.nvvm.fmax.ftz.nan.xorsign.abs.f" => "__nvvm_fmax_ftz_nan_xorsign_abs_f",
+    "llvm.nvvm.fmax.ftz.nan.xorsign.abs.f16" => "__nvvm_fmax_ftz_nan_xorsign_abs_f16",
+    "llvm.nvvm.fmax.ftz.nan.xorsign.abs.f16x2" => "__nvvm_fmax_ftz_nan_xorsign_abs_f16x2",
+    "llvm.nvvm.fmax.ftz.xorsign.abs.f" => "__nvvm_fmax_ftz_xorsign_abs_f",
+    "llvm.nvvm.fmax.ftz.xorsign.abs.f16" => "__nvvm_fmax_ftz_xorsign_abs_f16",
+    "llvm.nvvm.fmax.ftz.xorsign.abs.f16x2" => "__nvvm_fmax_ftz_xorsign_abs_f16x2",
+    "llvm.nvvm.fmax.nan.bf16" => "__nvvm_fmax_nan_bf16",
+    "llvm.nvvm.fmax.nan.bf16x2" => "__nvvm_fmax_nan_bf16x2",
+    "llvm.nvvm.fmax.nan.f" => "__nvvm_fmax_nan_f",
+    "llvm.nvvm.fmax.nan.f16" => "__nvvm_fmax_nan_f16",
+    "llvm.nvvm.fmax.nan.f16x2" => "__nvvm_fmax_nan_f16x2",
+    "llvm.nvvm.fmax.nan.xorsign.abs.bf16" => "__nvvm_fmax_nan_xorsign_abs_bf16",
+    "llvm.nvvm.fmax.nan.xorsign.abs.bf16x2" => "__nvvm_fmax_nan_xorsign_abs_bf16x2",
+    "llvm.nvvm.fmax.nan.xorsign.abs.f" => "__nvvm_fmax_nan_xorsign_abs_f",
+    "llvm.nvvm.fmax.nan.xorsign.abs.f16" => "__nvvm_fmax_nan_xorsign_abs_f16",
+    "llvm.nvvm.fmax.nan.xorsign.abs.f16x2" => "__nvvm_fmax_nan_xorsign_abs_f16x2",
+    "llvm.nvvm.fmax.xorsign.abs.bf16" => "__nvvm_fmax_xorsign_abs_bf16",
+    "llvm.nvvm.fmax.xorsign.abs.bf16x2" => "__nvvm_fmax_xorsign_abs_bf16x2",
+    "llvm.nvvm.fmax.xorsign.abs.f" => "__nvvm_fmax_xorsign_abs_f",
+    "llvm.nvvm.fmax.xorsign.abs.f16" => "__nvvm_fmax_xorsign_abs_f16",
+    "llvm.nvvm.fmax.xorsign.abs.f16x2" => "__nvvm_fmax_xorsign_abs_f16x2",
+    "llvm.nvvm.fmin.bf16" => "__nvvm_fmin_bf16",
+    "llvm.nvvm.fmin.bf16x2" => "__nvvm_fmin_bf16x2",
     "llvm.nvvm.fmin.d" => "__nvvm_fmin_d",
     "llvm.nvvm.fmin.f" => "__nvvm_fmin_f",
+    "llvm.nvvm.fmin.f16" => "__nvvm_fmin_f16",
+    "llvm.nvvm.fmin.f16x2" => "__nvvm_fmin_f16x2",
     "llvm.nvvm.fmin.ftz.f" => "__nvvm_fmin_ftz_f",
+    "llvm.nvvm.fmin.ftz.f16" => "__nvvm_fmin_ftz_f16",
+    "llvm.nvvm.fmin.ftz.f16x2" => "__nvvm_fmin_ftz_f16x2",
+    "llvm.nvvm.fmin.ftz.nan.f" => "__nvvm_fmin_ftz_nan_f",
+    "llvm.nvvm.fmin.ftz.nan.f16" => "__nvvm_fmin_ftz_nan_f16",
+    "llvm.nvvm.fmin.ftz.nan.f16x2" => "__nvvm_fmin_ftz_nan_f16x2",
+    "llvm.nvvm.fmin.ftz.nan.xorsign.abs.f" => "__nvvm_fmin_ftz_nan_xorsign_abs_f",
+    "llvm.nvvm.fmin.ftz.nan.xorsign.abs.f16" => "__nvvm_fmin_ftz_nan_xorsign_abs_f16",
+    "llvm.nvvm.fmin.ftz.nan.xorsign.abs.f16x2" => "__nvvm_fmin_ftz_nan_xorsign_abs_f16x2",
+    "llvm.nvvm.fmin.ftz.xorsign.abs.f" => "__nvvm_fmin_ftz_xorsign_abs_f",
+    "llvm.nvvm.fmin.ftz.xorsign.abs.f16" => "__nvvm_fmin_ftz_xorsign_abs_f16",
+    "llvm.nvvm.fmin.ftz.xorsign.abs.f16x2" => "__nvvm_fmin_ftz_xorsign_abs_f16x2",
+    "llvm.nvvm.fmin.nan.bf16" => "__nvvm_fmin_nan_bf16",
+    "llvm.nvvm.fmin.nan.bf16x2" => "__nvvm_fmin_nan_bf16x2",
+    "llvm.nvvm.fmin.nan.f" => "__nvvm_fmin_nan_f",
+    "llvm.nvvm.fmin.nan.f16" => "__nvvm_fmin_nan_f16",
+    "llvm.nvvm.fmin.nan.f16x2" => "__nvvm_fmin_nan_f16x2",
+    "llvm.nvvm.fmin.nan.xorsign.abs.bf16" => "__nvvm_fmin_nan_xorsign_abs_bf16",
+    "llvm.nvvm.fmin.nan.xorsign.abs.bf16x2" => "__nvvm_fmin_nan_xorsign_abs_bf16x2",
+    "llvm.nvvm.fmin.nan.xorsign.abs.f" => "__nvvm_fmin_nan_xorsign_abs_f",
+    "llvm.nvvm.fmin.nan.xorsign.abs.f16" => "__nvvm_fmin_nan_xorsign_abs_f16",
+    "llvm.nvvm.fmin.nan.xorsign.abs.f16x2" => "__nvvm_fmin_nan_xorsign_abs_f16x2",
+    "llvm.nvvm.fmin.xorsign.abs.bf16" => "__nvvm_fmin_xorsign_abs_bf16",
+    "llvm.nvvm.fmin.xorsign.abs.bf16x2" => "__nvvm_fmin_xorsign_abs_bf16x2",
+    "llvm.nvvm.fmin.xorsign.abs.f" => "__nvvm_fmin_xorsign_abs_f",
+    "llvm.nvvm.fmin.xorsign.abs.f16" => "__nvvm_fmin_xorsign_abs_f16",
+    "llvm.nvvm.fmin.xorsign.abs.f16x2" => "__nvvm_fmin_xorsign_abs_f16x2",
+    "llvm.nvvm.fns" => "__nvvm_fns",
     "llvm.nvvm.h2f" => "__nvvm_h2f",
     "llvm.nvvm.i2d.rm" => "__nvvm_i2d_rm",
     "llvm.nvvm.i2d.rn" => "__nvvm_i2d_rn",
@@ -2461,10 +3199,27 @@ match name {
     "llvm.nvvm.ll2f.rp" => "__nvvm_ll2f_rp",
     "llvm.nvvm.ll2f.rz" => "__nvvm_ll2f_rz",
     "llvm.nvvm.lohi.i2d" => "__nvvm_lohi_i2d",
+    "llvm.nvvm.match.any.sync.i32" => "__nvvm_match_any_sync_i32",
+    "llvm.nvvm.match.any.sync.i64" => "__nvvm_match_any_sync_i64",
     "llvm.nvvm.max.i" => "__nvvm_max_i",
     "llvm.nvvm.max.ll" => "__nvvm_max_ll",
     "llvm.nvvm.max.ui" => "__nvvm_max_ui",
     "llvm.nvvm.max.ull" => "__nvvm_max_ull",
+    "llvm.nvvm.mbarrier.arrive" => "__nvvm_mbarrier_arrive",
+    "llvm.nvvm.mbarrier.arrive.drop" => "__nvvm_mbarrier_arrive_drop",
+    "llvm.nvvm.mbarrier.arrive.drop.noComplete" => "__nvvm_mbarrier_arrive_drop_noComplete",
+    "llvm.nvvm.mbarrier.arrive.drop.noComplete.shared" => "__nvvm_mbarrier_arrive_drop_noComplete_shared",
+    "llvm.nvvm.mbarrier.arrive.drop.shared" => "__nvvm_mbarrier_arrive_drop_shared",
+    "llvm.nvvm.mbarrier.arrive.noComplete" => "__nvvm_mbarrier_arrive_noComplete",
+    "llvm.nvvm.mbarrier.arrive.noComplete.shared" => "__nvvm_mbarrier_arrive_noComplete_shared",
+    "llvm.nvvm.mbarrier.arrive.shared" => "__nvvm_mbarrier_arrive_shared",
+    "llvm.nvvm.mbarrier.init" => "__nvvm_mbarrier_init",
+    "llvm.nvvm.mbarrier.init.shared" => "__nvvm_mbarrier_init_shared",
+    "llvm.nvvm.mbarrier.inval" => "__nvvm_mbarrier_inval",
+    "llvm.nvvm.mbarrier.inval.shared" => "__nvvm_mbarrier_inval_shared",
+    "llvm.nvvm.mbarrier.pending.count" => "__nvvm_mbarrier_pending_count",
+    "llvm.nvvm.mbarrier.test.wait" => "__nvvm_mbarrier_test_wait",
+    "llvm.nvvm.mbarrier.test.wait.shared" => "__nvvm_mbarrier_test_wait_shared",
     "llvm.nvvm.membar.cta" => "__nvvm_membar_cta",
     "llvm.nvvm.membar.gl" => "__nvvm_membar_gl",
     "llvm.nvvm.membar.sys" => "__nvvm_membar_sys",
@@ -2490,10 +3245,13 @@ match name {
     "llvm.nvvm.mulhi.ll" => "__nvvm_mulhi_ll",
     "llvm.nvvm.mulhi.ui" => "__nvvm_mulhi_ui",
     "llvm.nvvm.mulhi.ull" => "__nvvm_mulhi_ull",
+    "llvm.nvvm.neg.bf16" => "__nvvm_neg_bf16",
+    "llvm.nvvm.neg.bf16x2" => "__nvvm_neg_bf16x2",
     "llvm.nvvm.popc.i" => "__nvvm_popc_i",
     "llvm.nvvm.popc.ll" => "__nvvm_popc_ll",
     "llvm.nvvm.prmt" => "__nvvm_prmt",
     "llvm.nvvm.rcp.approx.ftz.d" => "__nvvm_rcp_approx_ftz_d",
+    "llvm.nvvm.rcp.approx.ftz.f" => "__nvvm_rcp_approx_ftz_f",
     "llvm.nvvm.rcp.rm.d" => "__nvvm_rcp_rm_d",
     "llvm.nvvm.rcp.rm.f" => "__nvvm_rcp_rm_f",
     "llvm.nvvm.rcp.rm.ftz.f" => "__nvvm_rcp_rm_ftz_f",
@@ -2506,8 +3264,11 @@ match name {
     "llvm.nvvm.rcp.rz.d" => "__nvvm_rcp_rz_d",
     "llvm.nvvm.rcp.rz.f" => "__nvvm_rcp_rz_f",
     "llvm.nvvm.rcp.rz.ftz.f" => "__nvvm_rcp_rz_ftz_f",
-    "llvm.nvvm.read.ptx.sreg.clock" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.clock64" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.clock" => "__nvvm_read_ptx_sreg_clock",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.clock" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.clock64" => "__nvvm_read_ptx_sreg_clock64",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.clock64" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.ctaid.w" => "__nvvm_read_ptx_sreg_ctaid_w",
     "llvm.nvvm.read.ptx.sreg.ctaid.x" => "__nvvm_read_ptx_sreg_ctaid_x",
     "llvm.nvvm.read.ptx.sreg.ctaid.y" => "__nvvm_read_ptx_sreg_ctaid_y",
     "llvm.nvvm.read.ptx.sreg.ctaid.z" => "__nvvm_read_ptx_sreg_ctaid_z",
@@ -2543,32 +3304,58 @@ match name {
     "llvm.nvvm.read.ptx.sreg.envreg7" => "__nvvm_read_ptx_sreg_envreg7",
     "llvm.nvvm.read.ptx.sreg.envreg8" => "__nvvm_read_ptx_sreg_envreg8",
     "llvm.nvvm.read.ptx.sreg.envreg9" => "__nvvm_read_ptx_sreg_envreg9",
-    "llvm.nvvm.read.ptx.sreg.gridid" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.laneid" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.lanemask.eq" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.lanemask.ge" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.lanemask.gt" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.lanemask.le" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.lanemask.lt" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.gridid" => "__nvvm_read_ptx_sreg_gridid",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.gridid" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.laneid" => "__nvvm_read_ptx_sreg_laneid",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.laneid" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.lanemask.eq" => "__nvvm_read_ptx_sreg_lanemask_eq",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.lanemask.eq" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.lanemask.ge" => "__nvvm_read_ptx_sreg_lanemask_ge",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.lanemask.ge" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.lanemask.gt" => "__nvvm_read_ptx_sreg_lanemask_gt",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.lanemask.gt" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.lanemask.le" => "__nvvm_read_ptx_sreg_lanemask_le",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.lanemask.le" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.lanemask.lt" => "__nvvm_read_ptx_sreg_lanemask_lt",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.lanemask.lt" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.nctaid.w" => "__nvvm_read_ptx_sreg_nctaid_w",
     "llvm.nvvm.read.ptx.sreg.nctaid.x" => "__nvvm_read_ptx_sreg_nctaid_x",
     "llvm.nvvm.read.ptx.sreg.nctaid.y" => "__nvvm_read_ptx_sreg_nctaid_y",
     "llvm.nvvm.read.ptx.sreg.nctaid.z" => "__nvvm_read_ptx_sreg_nctaid_z",
-    "llvm.nvvm.read.ptx.sreg.nsmid" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.nsmid" => "__nvvm_read_ptx_sreg_nsmid",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.nsmid" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.ntid.w" => "__nvvm_read_ptx_sreg_ntid_w",
     "llvm.nvvm.read.ptx.sreg.ntid.x" => "__nvvm_read_ptx_sreg_ntid_x",
     "llvm.nvvm.read.ptx.sreg.ntid.y" => "__nvvm_read_ptx_sreg_ntid_y",
     "llvm.nvvm.read.ptx.sreg.ntid.z" => "__nvvm_read_ptx_sreg_ntid_z",
-    "llvm.nvvm.read.ptx.sreg.nwarpid" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.pm0" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.pm1" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.pm2" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.pm3" => "__nvvm_read_ptx_sreg_",
-    "llvm.nvvm.read.ptx.sreg.smid" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.nwarpid" => "__nvvm_read_ptx_sreg_nwarpid",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.nwarpid" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.pm0" => "__nvvm_read_ptx_sreg_pm0",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.pm0" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.pm1" => "__nvvm_read_ptx_sreg_pm1",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.pm1" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.pm2" => "__nvvm_read_ptx_sreg_pm2",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.pm2" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.pm3" => "__nvvm_read_ptx_sreg_pm3",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.pm3" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.smid" => "__nvvm_read_ptx_sreg_smid",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.smid" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.tid.w" => "__nvvm_read_ptx_sreg_tid_w",
     "llvm.nvvm.read.ptx.sreg.tid.x" => "__nvvm_read_ptx_sreg_tid_x",
     "llvm.nvvm.read.ptx.sreg.tid.y" => "__nvvm_read_ptx_sreg_tid_y",
     "llvm.nvvm.read.ptx.sreg.tid.z" => "__nvvm_read_ptx_sreg_tid_z",
-    "llvm.nvvm.read.ptx.sreg.warpid" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.read.ptx.sreg.warpid" => "__nvvm_read_ptx_sreg_warpid",
+    // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.warpid" => "__nvvm_read_ptx_sreg_",
     "llvm.nvvm.read.ptx.sreg.warpsize" => "__nvvm_read_ptx_sreg_warpsize",
     // [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.warpsize" => "__nvvm_read_ptx_sreg_",
+    "llvm.nvvm.redux.sync.add" => "__nvvm_redux_sync_add",
+    "llvm.nvvm.redux.sync.and" => "__nvvm_redux_sync_and",
+    "llvm.nvvm.redux.sync.max" => "__nvvm_redux_sync_max",
+    "llvm.nvvm.redux.sync.min" => "__nvvm_redux_sync_min",
+    "llvm.nvvm.redux.sync.or" => "__nvvm_redux_sync_or",
+    "llvm.nvvm.redux.sync.umax" => "__nvvm_redux_sync_umax",
+    "llvm.nvvm.redux.sync.umin" => "__nvvm_redux_sync_umin",
+    "llvm.nvvm.redux.sync.xor" => "__nvvm_redux_sync_xor",
     "llvm.nvvm.rotate.b32" => "__nvvm_rotate_b32",
     "llvm.nvvm.rotate.b64" => "__nvvm_rotate_b64",
     "llvm.nvvm.rotate.right.b64" => "__nvvm_rotate_right_b64",
@@ -2589,6 +3376,14 @@ match name {
     "llvm.nvvm.shfl.down.i32" => "__nvvm_shfl_down_i32",
     "llvm.nvvm.shfl.idx.f32" => "__nvvm_shfl_idx_f32",
     "llvm.nvvm.shfl.idx.i32" => "__nvvm_shfl_idx_i32",
+    "llvm.nvvm.shfl.sync.bfly.f32" => "__nvvm_shfl_sync_bfly_f32",
+    "llvm.nvvm.shfl.sync.bfly.i32" => "__nvvm_shfl_sync_bfly_i32",
+    "llvm.nvvm.shfl.sync.down.f32" => "__nvvm_shfl_sync_down_f32",
+    "llvm.nvvm.shfl.sync.down.i32" => "__nvvm_shfl_sync_down_i32",
+    "llvm.nvvm.shfl.sync.idx.f32" => "__nvvm_shfl_sync_idx_f32",
+    "llvm.nvvm.shfl.sync.idx.i32" => "__nvvm_shfl_sync_idx_i32",
+    "llvm.nvvm.shfl.sync.up.f32" => "__nvvm_shfl_sync_up_f32",
+    "llvm.nvvm.shfl.sync.up.i32" => "__nvvm_shfl_sync_up_i32",
     "llvm.nvvm.shfl.up.f32" => "__nvvm_shfl_up_f32",
     "llvm.nvvm.shfl.up.i32" => "__nvvm_shfl_up_i32",
     "llvm.nvvm.sin.approx.f" => "__nvvm_sin_approx_f",
@@ -2852,6 +3647,14 @@ match name {
     "llvm.nvvm.ull2f.rn" => "__nvvm_ull2f_rn",
     "llvm.nvvm.ull2f.rp" => "__nvvm_ull2f_rp",
     "llvm.nvvm.ull2f.rz" => "__nvvm_ull2f_rz",
+    "llvm.nvvm.vote.all" => "__nvvm_vote_all",
+    "llvm.nvvm.vote.all.sync" => "__nvvm_vote_all_sync",
+    "llvm.nvvm.vote.any" => "__nvvm_vote_any",
+    "llvm.nvvm.vote.any.sync" => "__nvvm_vote_any_sync",
+    "llvm.nvvm.vote.ballot" => "__nvvm_vote_ballot",
+    "llvm.nvvm.vote.ballot.sync" => "__nvvm_vote_ballot_sync",
+    "llvm.nvvm.vote.uni" => "__nvvm_vote_uni",
+    "llvm.nvvm.vote.uni.sync" => "__nvvm_vote_uni_sync",
     // ppc
     "llvm.ppc.addex" => "__builtin_ppc_addex",
     "llvm.ppc.addf128.round.to.odd" => "__builtin_addf128_round_to_odd",
@@ -2881,6 +3684,10 @@ match name {
     "llvm.ppc.altivec.mtvsrhm" => "__builtin_altivec_mtvsrhm",
     "llvm.ppc.altivec.mtvsrqm" => "__builtin_altivec_mtvsrqm",
     "llvm.ppc.altivec.mtvsrwm" => "__builtin_altivec_mtvsrwm",
+    "llvm.ppc.altivec.vabsdub" => "__builtin_altivec_vabsdub",
+    "llvm.ppc.altivec.vabsduh" => "__builtin_altivec_vabsduh",
+    "llvm.ppc.altivec.vabsduw" => "__builtin_altivec_vabsduw",
+    "llvm.ppc.altivec.vaddcuq" => "__builtin_altivec_vaddcuq",
     "llvm.ppc.altivec.vaddcuw" => "__builtin_altivec_vaddcuw",
     "llvm.ppc.altivec.vaddecuq" => "__builtin_altivec_vaddecuq",
     "llvm.ppc.altivec.vaddeuqm" => "__builtin_altivec_vaddeuqm",
@@ -2963,6 +3770,12 @@ match name {
     "llvm.ppc.altivec.vctuxs" => "__builtin_altivec_vctuxs",
     "llvm.ppc.altivec.vctzdm" => "__builtin_altivec_vctzdm",
     "llvm.ppc.altivec.vctzlsbb" => "__builtin_altivec_vctzlsbb",
+    "llvm.ppc.altivec.vdivesd" => "__builtin_altivec_vdivesd",
+    "llvm.ppc.altivec.vdivesq" => "__builtin_altivec_vdivesq",
+    "llvm.ppc.altivec.vdivesw" => "__builtin_altivec_vdivesw",
+    "llvm.ppc.altivec.vdiveud" => "__builtin_altivec_vdiveud",
+    "llvm.ppc.altivec.vdiveuq" => "__builtin_altivec_vdiveuq",
+    "llvm.ppc.altivec.vdiveuw" => "__builtin_altivec_vdiveuw",
     "llvm.ppc.altivec.vexpandbm" => "__builtin_altivec_vexpandbm",
     "llvm.ppc.altivec.vexpanddm" => "__builtin_altivec_vexpanddm",
     "llvm.ppc.altivec.vexpandhm" => "__builtin_altivec_vexpandhm",
@@ -3036,15 +3849,23 @@ match name {
     "llvm.ppc.altivec.vmsumuhm" => "__builtin_altivec_vmsumuhm",
     "llvm.ppc.altivec.vmsumuhs" => "__builtin_altivec_vmsumuhs",
     "llvm.ppc.altivec.vmulesb" => "__builtin_altivec_vmulesb",
+    "llvm.ppc.altivec.vmulesd" => "__builtin_altivec_vmulesd",
     "llvm.ppc.altivec.vmulesh" => "__builtin_altivec_vmulesh",
     "llvm.ppc.altivec.vmulesw" => "__builtin_altivec_vmulesw",
     "llvm.ppc.altivec.vmuleub" => "__builtin_altivec_vmuleub",
+    "llvm.ppc.altivec.vmuleud" => "__builtin_altivec_vmuleud",
     "llvm.ppc.altivec.vmuleuh" => "__builtin_altivec_vmuleuh",
     "llvm.ppc.altivec.vmuleuw" => "__builtin_altivec_vmuleuw",
+    "llvm.ppc.altivec.vmulhsd" => "__builtin_altivec_vmulhsd",
+    "llvm.ppc.altivec.vmulhsw" => "__builtin_altivec_vmulhsw",
+    "llvm.ppc.altivec.vmulhud" => "__builtin_altivec_vmulhud",
+    "llvm.ppc.altivec.vmulhuw" => "__builtin_altivec_vmulhuw",
     "llvm.ppc.altivec.vmulosb" => "__builtin_altivec_vmulosb",
+    "llvm.ppc.altivec.vmulosd" => "__builtin_altivec_vmulosd",
     "llvm.ppc.altivec.vmulosh" => "__builtin_altivec_vmulosh",
     "llvm.ppc.altivec.vmulosw" => "__builtin_altivec_vmulosw",
     "llvm.ppc.altivec.vmuloub" => "__builtin_altivec_vmuloub",
+    "llvm.ppc.altivec.vmuloud" => "__builtin_altivec_vmuloud",
     "llvm.ppc.altivec.vmulouh" => "__builtin_altivec_vmulouh",
     "llvm.ppc.altivec.vmulouw" => "__builtin_altivec_vmulouw",
     "llvm.ppc.altivec.vnmsubfp" => "__builtin_altivec_vnmsubfp",
@@ -3071,8 +3892,14 @@ match name {
     "llvm.ppc.altivec.vrfiz" => "__builtin_altivec_vrfiz",
     "llvm.ppc.altivec.vrlb" => "__builtin_altivec_vrlb",
     "llvm.ppc.altivec.vrld" => "__builtin_altivec_vrld",
+    "llvm.ppc.altivec.vrldmi" => "__builtin_altivec_vrldmi",
+    "llvm.ppc.altivec.vrldnm" => "__builtin_altivec_vrldnm",
     "llvm.ppc.altivec.vrlh" => "__builtin_altivec_vrlh",
+    "llvm.ppc.altivec.vrlqmi" => "__builtin_altivec_vrlqmi",
+    "llvm.ppc.altivec.vrlqnm" => "__builtin_altivec_vrlqnm",
     "llvm.ppc.altivec.vrlw" => "__builtin_altivec_vrlw",
+    "llvm.ppc.altivec.vrlwmi" => "__builtin_altivec_vrlwmi",
+    "llvm.ppc.altivec.vrlwnm" => "__builtin_altivec_vrlwnm",
     "llvm.ppc.altivec.vrsqrtefp" => "__builtin_altivec_vrsqrtefp",
     "llvm.ppc.altivec.vsel" => "__builtin_altivec_vsel_4si",
     "llvm.ppc.altivec.vsl" => "__builtin_altivec_vsl",
@@ -3080,6 +3907,7 @@ match name {
     "llvm.ppc.altivec.vsldbi" => "__builtin_altivec_vsldbi",
     "llvm.ppc.altivec.vslh" => "__builtin_altivec_vslh",
     "llvm.ppc.altivec.vslo" => "__builtin_altivec_vslo",
+    "llvm.ppc.altivec.vslv" => "__builtin_altivec_vslv",
     "llvm.ppc.altivec.vslw" => "__builtin_altivec_vslw",
     "llvm.ppc.altivec.vsr" => "__builtin_altivec_vsr",
     "llvm.ppc.altivec.vsrab" => "__builtin_altivec_vsrab",
@@ -3089,6 +3917,7 @@ match name {
     "llvm.ppc.altivec.vsrdbi" => "__builtin_altivec_vsrdbi",
     "llvm.ppc.altivec.vsrh" => "__builtin_altivec_vsrh",
     "llvm.ppc.altivec.vsro" => "__builtin_altivec_vsro",
+    "llvm.ppc.altivec.vsrv" => "__builtin_altivec_vsrv",
     "llvm.ppc.altivec.vsrw" => "__builtin_altivec_vsrw",
     "llvm.ppc.altivec.vstribl" => "__builtin_altivec_vstribl",
     "llvm.ppc.altivec.vstribl.p" => "__builtin_altivec_vstribl_p",
@@ -3098,6 +3927,7 @@ match name {
     "llvm.ppc.altivec.vstrihl.p" => "__builtin_altivec_vstrihl_p",
     "llvm.ppc.altivec.vstrihr" => "__builtin_altivec_vstrihr",
     "llvm.ppc.altivec.vstrihr.p" => "__builtin_altivec_vstrihr_p",
+    "llvm.ppc.altivec.vsubcuq" => "__builtin_altivec_vsubcuq",
     "llvm.ppc.altivec.vsubcuw" => "__builtin_altivec_vsubcuw",
     "llvm.ppc.altivec.vsubecuq" => "__builtin_altivec_vsubecuq",
     "llvm.ppc.altivec.vsubeuqm" => "__builtin_altivec_vsubeuqm",
@@ -3165,6 +3995,8 @@ match name {
     "llvm.ppc.fmaf128.round.to.odd" => "__builtin_fmaf128_round_to_odd",
     "llvm.ppc.fmsub" => "__builtin_ppc_fmsub",
     "llvm.ppc.fmsubs" => "__builtin_ppc_fmsubs",
+    "llvm.ppc.fnabs" => "__builtin_ppc_fnabs",
+    "llvm.ppc.fnabss" => "__builtin_ppc_fnabss",
     "llvm.ppc.fnmadd" => "__builtin_ppc_fnmadd",
     "llvm.ppc.fnmadds" => "__builtin_ppc_fnmadds",
     "llvm.ppc.fre" => "__builtin_ppc_fre",
@@ -3341,8 +4173,24 @@ match name {
     "llvm.ppc.vsx.xvcmpgtdp.p" => "__builtin_vsx_xvcmpgtdp_p",
     "llvm.ppc.vsx.xvcmpgtsp" => "__builtin_vsx_xvcmpgtsp",
     "llvm.ppc.vsx.xvcmpgtsp.p" => "__builtin_vsx_xvcmpgtsp_p",
+    "llvm.ppc.vsx.xvcvbf16spn" => "__builtin_vsx_xvcvbf16spn",
+    "llvm.ppc.vsx.xvcvdpsp" => "__builtin_vsx_xvcvdpsp",
+    "llvm.ppc.vsx.xvcvdpsxws" => "__builtin_vsx_xvcvdpsxws",
+    "llvm.ppc.vsx.xvcvdpuxws" => "__builtin_vsx_xvcvdpuxws",
+    "llvm.ppc.vsx.xvcvhpsp" => "__builtin_vsx_xvcvhpsp",
+    "llvm.ppc.vsx.xvcvspbf16" => "__builtin_vsx_xvcvspbf16",
+    "llvm.ppc.vsx.xvcvspdp" => "__builtin_vsx_xvcvspdp",
+    "llvm.ppc.vsx.xvcvsphp" => "__builtin_vsx_xvcvsphp",
+    "llvm.ppc.vsx.xvcvspsxds" => "__builtin_vsx_xvcvspsxds",
+    "llvm.ppc.vsx.xvcvspuxds" => "__builtin_vsx_xvcvspuxds",
+    "llvm.ppc.vsx.xvcvsxdsp" => "__builtin_vsx_xvcvsxdsp",
+    "llvm.ppc.vsx.xvcvsxwdp" => "__builtin_vsx_xvcvsxwdp",
+    "llvm.ppc.vsx.xvcvuxdsp" => "__builtin_vsx_xvcvuxdsp",
+    "llvm.ppc.vsx.xvcvuxwdp" => "__builtin_vsx_xvcvuxwdp",
     "llvm.ppc.vsx.xvdivdp" => "__builtin_vsx_xvdivdp",
     "llvm.ppc.vsx.xvdivsp" => "__builtin_vsx_xvdivsp",
+    "llvm.ppc.vsx.xviexpdp" => "__builtin_vsx_xviexpdp",
+    "llvm.ppc.vsx.xviexpsp" => "__builtin_vsx_xviexpsp",
     "llvm.ppc.vsx.xvmaxdp" => "__builtin_vsx_xvmaxdp",
     "llvm.ppc.vsx.xvmaxsp" => "__builtin_vsx_xvmaxsp",
     "llvm.ppc.vsx.xvmindp" => "__builtin_vsx_xvmindp",
@@ -3351,10 +4199,28 @@ match name {
     "llvm.ppc.vsx.xvresp" => "__builtin_vsx_xvresp",
     "llvm.ppc.vsx.xvrsqrtedp" => "__builtin_vsx_xvrsqrtedp",
     "llvm.ppc.vsx.xvrsqrtesp" => "__builtin_vsx_xvrsqrtesp",
+    "llvm.ppc.vsx.xvtdivdp" => "__builtin_vsx_xvtdivdp",
+    "llvm.ppc.vsx.xvtdivsp" => "__builtin_vsx_xvtdivsp",
+    "llvm.ppc.vsx.xvtlsbb" => "__builtin_vsx_xvtlsbb",
+    "llvm.ppc.vsx.xvtsqrtdp" => "__builtin_vsx_xvtsqrtdp",
+    "llvm.ppc.vsx.xvtsqrtsp" => "__builtin_vsx_xvtsqrtsp",
+    "llvm.ppc.vsx.xvtstdcdp" => "__builtin_vsx_xvtstdcdp",
+    "llvm.ppc.vsx.xvtstdcsp" => "__builtin_vsx_xvtstdcsp",
+    "llvm.ppc.vsx.xvxexpdp" => "__builtin_vsx_xvxexpdp",
+    "llvm.ppc.vsx.xvxexpsp" => "__builtin_vsx_xvxexpsp",
+    "llvm.ppc.vsx.xvxsigdp" => "__builtin_vsx_xvxsigdp",
+    "llvm.ppc.vsx.xvxsigsp" => "__builtin_vsx_xvxsigsp",
     "llvm.ppc.vsx.xxblendvb" => "__builtin_vsx_xxblendvb",
     "llvm.ppc.vsx.xxblendvd" => "__builtin_vsx_xxblendvd",
     "llvm.ppc.vsx.xxblendvh" => "__builtin_vsx_xxblendvh",
     "llvm.ppc.vsx.xxblendvw" => "__builtin_vsx_xxblendvw",
+    "llvm.ppc.vsx.xxeval" => "__builtin_vsx_xxeval",
+    "llvm.ppc.vsx.xxextractuw" => "__builtin_vsx_xxextractuw",
+    "llvm.ppc.vsx.xxgenpcvbm" => "__builtin_vsx_xxgenpcvbm",
+    "llvm.ppc.vsx.xxgenpcvdm" => "__builtin_vsx_xxgenpcvdm",
+    "llvm.ppc.vsx.xxgenpcvhm" => "__builtin_vsx_xxgenpcvhm",
+    "llvm.ppc.vsx.xxgenpcvwm" => "__builtin_vsx_xxgenpcvwm",
+    "llvm.ppc.vsx.xxinsertw" => "__builtin_vsx_xxinsertw",
     "llvm.ppc.vsx.xxleqv" => "__builtin_vsx_xxleqv",
     "llvm.ppc.vsx.xxpermx" => "__builtin_vsx_xxpermx",
     // ptx
@@ -3376,6 +4242,19 @@ match name {
     "llvm.ptx.read.pm3" => "__builtin_ptx_read_pm3",
     "llvm.ptx.read.smid" => "__builtin_ptx_read_smid",
     "llvm.ptx.read.warpid" => "__builtin_ptx_read_warpid",
+    // r600
+    "llvm.r600.group.barrier" => "__builtin_r600_group_barrier",
+    "llvm.r600.implicitarg.ptr" => "__builtin_r600_implicitarg_ptr",
+    "llvm.r600.rat.store.typed" => "__builtin_r600_rat_store_typed",
+    "llvm.r600.read.global.size.x" => "__builtin_r600_read_global_size_x",
+    "llvm.r600.read.global.size.y" => "__builtin_r600_read_global_size_y",
+    "llvm.r600.read.global.size.z" => "__builtin_r600_read_global_size_z",
+    "llvm.r600.read.ngroups.x" => "__builtin_r600_read_ngroups_x",
+    "llvm.r600.read.ngroups.y" => "__builtin_r600_read_ngroups_y",
+    "llvm.r600.read.ngroups.z" => "__builtin_r600_read_ngroups_z",
+    "llvm.r600.read.tgid.x" => "__builtin_r600_read_tgid_x",
+    "llvm.r600.read.tgid.y" => "__builtin_r600_read_tgid_y",
+    "llvm.r600.read.tgid.z" => "__builtin_r600_read_tgid_z",
     // s390
     "llvm.s390.efpc" => "__builtin_s390_efpc",
     "llvm.s390.etnd" => "__builtin_tx_nesting_depth",
@@ -3383,29 +4262,1426 @@ match name {
     "llvm.s390.ppa.txassist" => "__builtin_tx_assist",
     "llvm.s390.sfpc" => "__builtin_s390_sfpc",
     "llvm.s390.tend" => "__builtin_tend",
+    "llvm.s390.vaccb" => "__builtin_s390_vaccb",
+    "llvm.s390.vacccq" => "__builtin_s390_vacccq",
+    "llvm.s390.vaccf" => "__builtin_s390_vaccf",
+    "llvm.s390.vaccg" => "__builtin_s390_vaccg",
+    "llvm.s390.vacch" => "__builtin_s390_vacch",
+    "llvm.s390.vaccq" => "__builtin_s390_vaccq",
+    "llvm.s390.vacq" => "__builtin_s390_vacq",
+    "llvm.s390.vaq" => "__builtin_s390_vaq",
+    "llvm.s390.vavgb" => "__builtin_s390_vavgb",
+    "llvm.s390.vavgf" => "__builtin_s390_vavgf",
+    "llvm.s390.vavgg" => "__builtin_s390_vavgg",
+    "llvm.s390.vavgh" => "__builtin_s390_vavgh",
+    "llvm.s390.vavglb" => "__builtin_s390_vavglb",
+    "llvm.s390.vavglf" => "__builtin_s390_vavglf",
+    "llvm.s390.vavglg" => "__builtin_s390_vavglg",
+    "llvm.s390.vavglh" => "__builtin_s390_vavglh",
+    "llvm.s390.vbperm" => "__builtin_s390_vbperm",
     "llvm.s390.vcfn" => "__builtin_s390_vcfn",
+    "llvm.s390.vcksm" => "__builtin_s390_vcksm",
     "llvm.s390.vclfnhs" => "__builtin_s390_vclfnhs",
     "llvm.s390.vclfnls" => "__builtin_s390_vclfnls",
     "llvm.s390.vcnf" => "__builtin_s390_vcnf",
     "llvm.s390.vcrnfs" => "__builtin_s390_vcrnfs",
+    "llvm.s390.verimb" => "__builtin_s390_verimb",
+    "llvm.s390.verimf" => "__builtin_s390_verimf",
+    "llvm.s390.verimg" => "__builtin_s390_verimg",
+    "llvm.s390.verimh" => "__builtin_s390_verimh",
+    "llvm.s390.verllb" => "__builtin_s390_verllb",
+    "llvm.s390.verllf" => "__builtin_s390_verllf",
+    "llvm.s390.verllg" => "__builtin_s390_verllg",
+    "llvm.s390.verllh" => "__builtin_s390_verllh",
+    "llvm.s390.verllvb" => "__builtin_s390_verllvb",
+    "llvm.s390.verllvf" => "__builtin_s390_verllvf",
+    "llvm.s390.verllvg" => "__builtin_s390_verllvg",
+    "llvm.s390.verllvh" => "__builtin_s390_verllvh",
+    "llvm.s390.vfaeb" => "__builtin_s390_vfaeb",
+    "llvm.s390.vfaef" => "__builtin_s390_vfaef",
+    "llvm.s390.vfaeh" => "__builtin_s390_vfaeh",
+    "llvm.s390.vfaezb" => "__builtin_s390_vfaezb",
+    "llvm.s390.vfaezf" => "__builtin_s390_vfaezf",
+    "llvm.s390.vfaezh" => "__builtin_s390_vfaezh",
+    "llvm.s390.vfeeb" => "__builtin_s390_vfeeb",
+    "llvm.s390.vfeef" => "__builtin_s390_vfeef",
+    "llvm.s390.vfeeh" => "__builtin_s390_vfeeh",
+    "llvm.s390.vfeezb" => "__builtin_s390_vfeezb",
+    "llvm.s390.vfeezf" => "__builtin_s390_vfeezf",
+    "llvm.s390.vfeezh" => "__builtin_s390_vfeezh",
+    "llvm.s390.vfeneb" => "__builtin_s390_vfeneb",
+    "llvm.s390.vfenef" => "__builtin_s390_vfenef",
+    "llvm.s390.vfeneh" => "__builtin_s390_vfeneh",
+    "llvm.s390.vfenezb" => "__builtin_s390_vfenezb",
+    "llvm.s390.vfenezf" => "__builtin_s390_vfenezf",
+    "llvm.s390.vfenezh" => "__builtin_s390_vfenezh",
+    "llvm.s390.vgfmab" => "__builtin_s390_vgfmab",
+    "llvm.s390.vgfmaf" => "__builtin_s390_vgfmaf",
+    "llvm.s390.vgfmag" => "__builtin_s390_vgfmag",
+    "llvm.s390.vgfmah" => "__builtin_s390_vgfmah",
+    "llvm.s390.vgfmb" => "__builtin_s390_vgfmb",
+    "llvm.s390.vgfmf" => "__builtin_s390_vgfmf",
+    "llvm.s390.vgfmg" => "__builtin_s390_vgfmg",
+    "llvm.s390.vgfmh" => "__builtin_s390_vgfmh",
+    "llvm.s390.vistrb" => "__builtin_s390_vistrb",
+    "llvm.s390.vistrf" => "__builtin_s390_vistrf",
+    "llvm.s390.vistrh" => "__builtin_s390_vistrh",
     "llvm.s390.vlbb" => "__builtin_s390_vlbb",
     "llvm.s390.vll" => "__builtin_s390_vll",
     "llvm.s390.vlrl" => "__builtin_s390_vlrl",
+    "llvm.s390.vmaeb" => "__builtin_s390_vmaeb",
+    "llvm.s390.vmaef" => "__builtin_s390_vmaef",
+    "llvm.s390.vmaeh" => "__builtin_s390_vmaeh",
+    "llvm.s390.vmahb" => "__builtin_s390_vmahb",
+    "llvm.s390.vmahf" => "__builtin_s390_vmahf",
+    "llvm.s390.vmahh" => "__builtin_s390_vmahh",
+    "llvm.s390.vmaleb" => "__builtin_s390_vmaleb",
+    "llvm.s390.vmalef" => "__builtin_s390_vmalef",
+    "llvm.s390.vmaleh" => "__builtin_s390_vmaleh",
+    "llvm.s390.vmalhb" => "__builtin_s390_vmalhb",
+    "llvm.s390.vmalhf" => "__builtin_s390_vmalhf",
+    "llvm.s390.vmalhh" => "__builtin_s390_vmalhh",
+    "llvm.s390.vmalob" => "__builtin_s390_vmalob",
+    "llvm.s390.vmalof" => "__builtin_s390_vmalof",
+    "llvm.s390.vmaloh" => "__builtin_s390_vmaloh",
+    "llvm.s390.vmaob" => "__builtin_s390_vmaob",
+    "llvm.s390.vmaof" => "__builtin_s390_vmaof",
+    "llvm.s390.vmaoh" => "__builtin_s390_vmaoh",
+    "llvm.s390.vmeb" => "__builtin_s390_vmeb",
+    "llvm.s390.vmef" => "__builtin_s390_vmef",
+    "llvm.s390.vmeh" => "__builtin_s390_vmeh",
+    "llvm.s390.vmhb" => "__builtin_s390_vmhb",
+    "llvm.s390.vmhf" => "__builtin_s390_vmhf",
+    "llvm.s390.vmhh" => "__builtin_s390_vmhh",
+    "llvm.s390.vmleb" => "__builtin_s390_vmleb",
+    "llvm.s390.vmlef" => "__builtin_s390_vmlef",
+    "llvm.s390.vmleh" => "__builtin_s390_vmleh",
+    "llvm.s390.vmlhb" => "__builtin_s390_vmlhb",
+    "llvm.s390.vmlhf" => "__builtin_s390_vmlhf",
+    "llvm.s390.vmlhh" => "__builtin_s390_vmlhh",
+    "llvm.s390.vmlob" => "__builtin_s390_vmlob",
+    "llvm.s390.vmlof" => "__builtin_s390_vmlof",
+    "llvm.s390.vmloh" => "__builtin_s390_vmloh",
+    "llvm.s390.vmob" => "__builtin_s390_vmob",
+    "llvm.s390.vmof" => "__builtin_s390_vmof",
+    "llvm.s390.vmoh" => "__builtin_s390_vmoh",
     "llvm.s390.vmslg" => "__builtin_s390_vmslg",
     "llvm.s390.vpdi" => "__builtin_s390_vpdi",
     "llvm.s390.vperm" => "__builtin_s390_vperm",
+    "llvm.s390.vpklsf" => "__builtin_s390_vpklsf",
+    "llvm.s390.vpklsg" => "__builtin_s390_vpklsg",
+    "llvm.s390.vpklsh" => "__builtin_s390_vpklsh",
+    "llvm.s390.vpksf" => "__builtin_s390_vpksf",
+    "llvm.s390.vpksg" => "__builtin_s390_vpksg",
+    "llvm.s390.vpksh" => "__builtin_s390_vpksh",
+    "llvm.s390.vsbcbiq" => "__builtin_s390_vsbcbiq",
+    "llvm.s390.vsbiq" => "__builtin_s390_vsbiq",
+    "llvm.s390.vscbib" => "__builtin_s390_vscbib",
+    "llvm.s390.vscbif" => "__builtin_s390_vscbif",
+    "llvm.s390.vscbig" => "__builtin_s390_vscbig",
+    "llvm.s390.vscbih" => "__builtin_s390_vscbih",
+    "llvm.s390.vscbiq" => "__builtin_s390_vscbiq",
+    "llvm.s390.vsl" => "__builtin_s390_vsl",
+    "llvm.s390.vslb" => "__builtin_s390_vslb",
     "llvm.s390.vsld" => "__builtin_s390_vsld",
     "llvm.s390.vsldb" => "__builtin_s390_vsldb",
+    "llvm.s390.vsq" => "__builtin_s390_vsq",
+    "llvm.s390.vsra" => "__builtin_s390_vsra",
+    "llvm.s390.vsrab" => "__builtin_s390_vsrab",
     "llvm.s390.vsrd" => "__builtin_s390_vsrd",
+    "llvm.s390.vsrl" => "__builtin_s390_vsrl",
+    "llvm.s390.vsrlb" => "__builtin_s390_vsrlb",
     "llvm.s390.vstl" => "__builtin_s390_vstl",
+    "llvm.s390.vstrcb" => "__builtin_s390_vstrcb",
+    "llvm.s390.vstrcf" => "__builtin_s390_vstrcf",
+    "llvm.s390.vstrch" => "__builtin_s390_vstrch",
+    "llvm.s390.vstrczb" => "__builtin_s390_vstrczb",
+    "llvm.s390.vstrczf" => "__builtin_s390_vstrczf",
+    "llvm.s390.vstrczh" => "__builtin_s390_vstrczh",
     "llvm.s390.vstrl" => "__builtin_s390_vstrl",
+    "llvm.s390.vsumb" => "__builtin_s390_vsumb",
+    "llvm.s390.vsumgf" => "__builtin_s390_vsumgf",
+    "llvm.s390.vsumgh" => "__builtin_s390_vsumgh",
+    "llvm.s390.vsumh" => "__builtin_s390_vsumh",
+    "llvm.s390.vsumqf" => "__builtin_s390_vsumqf",
+    "llvm.s390.vsumqg" => "__builtin_s390_vsumqg",
+    "llvm.s390.vtm" => "__builtin_s390_vtm",
+    "llvm.s390.vuphb" => "__builtin_s390_vuphb",
+    "llvm.s390.vuphf" => "__builtin_s390_vuphf",
+    "llvm.s390.vuphh" => "__builtin_s390_vuphh",
+    "llvm.s390.vuplb" => "__builtin_s390_vuplb",
+    "llvm.s390.vuplf" => "__builtin_s390_vuplf",
+    "llvm.s390.vuplhb" => "__builtin_s390_vuplhb",
+    "llvm.s390.vuplhf" => "__builtin_s390_vuplhf",
+    "llvm.s390.vuplhh" => "__builtin_s390_vuplhh",
+    "llvm.s390.vuplhw" => "__builtin_s390_vuplhw",
+    "llvm.s390.vupllb" => "__builtin_s390_vupllb",
+    "llvm.s390.vupllf" => "__builtin_s390_vupllf",
+    "llvm.s390.vupllh" => "__builtin_s390_vupllh",
     // ve
+    "llvm.ve.vl.andm.MMM" => "__builtin_ve_vl_andm_MMM",
+    "llvm.ve.vl.andm.mmm" => "__builtin_ve_vl_andm_mmm",
+    "llvm.ve.vl.eqvm.MMM" => "__builtin_ve_vl_eqvm_MMM",
+    "llvm.ve.vl.eqvm.mmm" => "__builtin_ve_vl_eqvm_mmm",
     "llvm.ve.vl.extract.vm512l" => "__builtin_ve_vl_extract_vm512l",
     "llvm.ve.vl.extract.vm512u" => "__builtin_ve_vl_extract_vm512u",
+    "llvm.ve.vl.fencec.s" => "__builtin_ve_vl_fencec_s",
+    "llvm.ve.vl.fencei" => "__builtin_ve_vl_fencei",
+    "llvm.ve.vl.fencem.s" => "__builtin_ve_vl_fencem_s",
+    "llvm.ve.vl.fidcr.sss" => "__builtin_ve_vl_fidcr_sss",
     "llvm.ve.vl.insert.vm512l" => "__builtin_ve_vl_insert_vm512l",
     "llvm.ve.vl.insert.vm512u" => "__builtin_ve_vl_insert_vm512u",
+    "llvm.ve.vl.lcr.sss" => "__builtin_ve_vl_lcr_sss",
+    "llvm.ve.vl.lsv.vvss" => "__builtin_ve_vl_lsv_vvss",
+    "llvm.ve.vl.lvm.MMss" => "__builtin_ve_vl_lvm_MMss",
+    "llvm.ve.vl.lvm.mmss" => "__builtin_ve_vl_lvm_mmss",
+    "llvm.ve.vl.lvsd.svs" => "__builtin_ve_vl_lvsd_svs",
+    "llvm.ve.vl.lvsl.svs" => "__builtin_ve_vl_lvsl_svs",
+    "llvm.ve.vl.lvss.svs" => "__builtin_ve_vl_lvss_svs",
+    "llvm.ve.vl.lzvm.sml" => "__builtin_ve_vl_lzvm_sml",
+    "llvm.ve.vl.negm.MM" => "__builtin_ve_vl_negm_MM",
+    "llvm.ve.vl.negm.mm" => "__builtin_ve_vl_negm_mm",
+    "llvm.ve.vl.nndm.MMM" => "__builtin_ve_vl_nndm_MMM",
+    "llvm.ve.vl.nndm.mmm" => "__builtin_ve_vl_nndm_mmm",
+    "llvm.ve.vl.orm.MMM" => "__builtin_ve_vl_orm_MMM",
+    "llvm.ve.vl.orm.mmm" => "__builtin_ve_vl_orm_mmm",
     "llvm.ve.vl.pack.f32a" => "__builtin_ve_vl_pack_f32a",
     "llvm.ve.vl.pack.f32p" => "__builtin_ve_vl_pack_f32p",
+    "llvm.ve.vl.pcvm.sml" => "__builtin_ve_vl_pcvm_sml",
+    "llvm.ve.vl.pfchv.ssl" => "__builtin_ve_vl_pfchv_ssl",
+    "llvm.ve.vl.pfchvnc.ssl" => "__builtin_ve_vl_pfchvnc_ssl",
+    "llvm.ve.vl.pvadds.vsvMvl" => "__builtin_ve_vl_pvadds_vsvMvl",
+    "llvm.ve.vl.pvadds.vsvl" => "__builtin_ve_vl_pvadds_vsvl",
+    "llvm.ve.vl.pvadds.vsvvl" => "__builtin_ve_vl_pvadds_vsvvl",
+    "llvm.ve.vl.pvadds.vvvMvl" => "__builtin_ve_vl_pvadds_vvvMvl",
+    "llvm.ve.vl.pvadds.vvvl" => "__builtin_ve_vl_pvadds_vvvl",
+    "llvm.ve.vl.pvadds.vvvvl" => "__builtin_ve_vl_pvadds_vvvvl",
+    "llvm.ve.vl.pvaddu.vsvMvl" => "__builtin_ve_vl_pvaddu_vsvMvl",
+    "llvm.ve.vl.pvaddu.vsvl" => "__builtin_ve_vl_pvaddu_vsvl",
+    "llvm.ve.vl.pvaddu.vsvvl" => "__builtin_ve_vl_pvaddu_vsvvl",
+    "llvm.ve.vl.pvaddu.vvvMvl" => "__builtin_ve_vl_pvaddu_vvvMvl",
+    "llvm.ve.vl.pvaddu.vvvl" => "__builtin_ve_vl_pvaddu_vvvl",
+    "llvm.ve.vl.pvaddu.vvvvl" => "__builtin_ve_vl_pvaddu_vvvvl",
+    "llvm.ve.vl.pvand.vsvMvl" => "__builtin_ve_vl_pvand_vsvMvl",
+    "llvm.ve.vl.pvand.vsvl" => "__builtin_ve_vl_pvand_vsvl",
+    "llvm.ve.vl.pvand.vsvvl" => "__builtin_ve_vl_pvand_vsvvl",
+    "llvm.ve.vl.pvand.vvvMvl" => "__builtin_ve_vl_pvand_vvvMvl",
+    "llvm.ve.vl.pvand.vvvl" => "__builtin_ve_vl_pvand_vvvl",
+    "llvm.ve.vl.pvand.vvvvl" => "__builtin_ve_vl_pvand_vvvvl",
+    "llvm.ve.vl.pvbrd.vsMvl" => "__builtin_ve_vl_pvbrd_vsMvl",
+    "llvm.ve.vl.pvbrd.vsl" => "__builtin_ve_vl_pvbrd_vsl",
+    "llvm.ve.vl.pvbrd.vsvl" => "__builtin_ve_vl_pvbrd_vsvl",
+    "llvm.ve.vl.pvbrv.vvMvl" => "__builtin_ve_vl_pvbrv_vvMvl",
+    "llvm.ve.vl.pvbrv.vvl" => "__builtin_ve_vl_pvbrv_vvl",
+    "llvm.ve.vl.pvbrv.vvvl" => "__builtin_ve_vl_pvbrv_vvvl",
+    "llvm.ve.vl.pvbrvlo.vvl" => "__builtin_ve_vl_pvbrvlo_vvl",
+    "llvm.ve.vl.pvbrvlo.vvmvl" => "__builtin_ve_vl_pvbrvlo_vvmvl",
+    "llvm.ve.vl.pvbrvlo.vvvl" => "__builtin_ve_vl_pvbrvlo_vvvl",
+    "llvm.ve.vl.pvbrvup.vvl" => "__builtin_ve_vl_pvbrvup_vvl",
+    "llvm.ve.vl.pvbrvup.vvmvl" => "__builtin_ve_vl_pvbrvup_vvmvl",
+    "llvm.ve.vl.pvbrvup.vvvl" => "__builtin_ve_vl_pvbrvup_vvvl",
+    "llvm.ve.vl.pvcmps.vsvMvl" => "__builtin_ve_vl_pvcmps_vsvMvl",
+    "llvm.ve.vl.pvcmps.vsvl" => "__builtin_ve_vl_pvcmps_vsvl",
+    "llvm.ve.vl.pvcmps.vsvvl" => "__builtin_ve_vl_pvcmps_vsvvl",
+    "llvm.ve.vl.pvcmps.vvvMvl" => "__builtin_ve_vl_pvcmps_vvvMvl",
+    "llvm.ve.vl.pvcmps.vvvl" => "__builtin_ve_vl_pvcmps_vvvl",
+    "llvm.ve.vl.pvcmps.vvvvl" => "__builtin_ve_vl_pvcmps_vvvvl",
+    "llvm.ve.vl.pvcmpu.vsvMvl" => "__builtin_ve_vl_pvcmpu_vsvMvl",
+    "llvm.ve.vl.pvcmpu.vsvl" => "__builtin_ve_vl_pvcmpu_vsvl",
+    "llvm.ve.vl.pvcmpu.vsvvl" => "__builtin_ve_vl_pvcmpu_vsvvl",
+    "llvm.ve.vl.pvcmpu.vvvMvl" => "__builtin_ve_vl_pvcmpu_vvvMvl",
+    "llvm.ve.vl.pvcmpu.vvvl" => "__builtin_ve_vl_pvcmpu_vvvl",
+    "llvm.ve.vl.pvcmpu.vvvvl" => "__builtin_ve_vl_pvcmpu_vvvvl",
+    "llvm.ve.vl.pvcvtsw.vvl" => "__builtin_ve_vl_pvcvtsw_vvl",
+    "llvm.ve.vl.pvcvtsw.vvvl" => "__builtin_ve_vl_pvcvtsw_vvvl",
+    "llvm.ve.vl.pvcvtws.vvMvl" => "__builtin_ve_vl_pvcvtws_vvMvl",
+    "llvm.ve.vl.pvcvtws.vvl" => "__builtin_ve_vl_pvcvtws_vvl",
+    "llvm.ve.vl.pvcvtws.vvvl" => "__builtin_ve_vl_pvcvtws_vvvl",
+    "llvm.ve.vl.pvcvtwsrz.vvMvl" => "__builtin_ve_vl_pvcvtwsrz_vvMvl",
+    "llvm.ve.vl.pvcvtwsrz.vvl" => "__builtin_ve_vl_pvcvtwsrz_vvl",
+    "llvm.ve.vl.pvcvtwsrz.vvvl" => "__builtin_ve_vl_pvcvtwsrz_vvvl",
+    "llvm.ve.vl.pveqv.vsvMvl" => "__builtin_ve_vl_pveqv_vsvMvl",
+    "llvm.ve.vl.pveqv.vsvl" => "__builtin_ve_vl_pveqv_vsvl",
+    "llvm.ve.vl.pveqv.vsvvl" => "__builtin_ve_vl_pveqv_vsvvl",
+    "llvm.ve.vl.pveqv.vvvMvl" => "__builtin_ve_vl_pveqv_vvvMvl",
+    "llvm.ve.vl.pveqv.vvvl" => "__builtin_ve_vl_pveqv_vvvl",
+    "llvm.ve.vl.pveqv.vvvvl" => "__builtin_ve_vl_pveqv_vvvvl",
+    "llvm.ve.vl.pvfadd.vsvMvl" => "__builtin_ve_vl_pvfadd_vsvMvl",
+    "llvm.ve.vl.pvfadd.vsvl" => "__builtin_ve_vl_pvfadd_vsvl",
+    "llvm.ve.vl.pvfadd.vsvvl" => "__builtin_ve_vl_pvfadd_vsvvl",
+    "llvm.ve.vl.pvfadd.vvvMvl" => "__builtin_ve_vl_pvfadd_vvvMvl",
+    "llvm.ve.vl.pvfadd.vvvl" => "__builtin_ve_vl_pvfadd_vvvl",
+    "llvm.ve.vl.pvfadd.vvvvl" => "__builtin_ve_vl_pvfadd_vvvvl",
+    "llvm.ve.vl.pvfcmp.vsvMvl" => "__builtin_ve_vl_pvfcmp_vsvMvl",
+    "llvm.ve.vl.pvfcmp.vsvl" => "__builtin_ve_vl_pvfcmp_vsvl",
+    "llvm.ve.vl.pvfcmp.vsvvl" => "__builtin_ve_vl_pvfcmp_vsvvl",
+    "llvm.ve.vl.pvfcmp.vvvMvl" => "__builtin_ve_vl_pvfcmp_vvvMvl",
+    "llvm.ve.vl.pvfcmp.vvvl" => "__builtin_ve_vl_pvfcmp_vvvl",
+    "llvm.ve.vl.pvfcmp.vvvvl" => "__builtin_ve_vl_pvfcmp_vvvvl",
+    "llvm.ve.vl.pvfmad.vsvvMvl" => "__builtin_ve_vl_pvfmad_vsvvMvl",
+    "llvm.ve.vl.pvfmad.vsvvl" => "__builtin_ve_vl_pvfmad_vsvvl",
+    "llvm.ve.vl.pvfmad.vsvvvl" => "__builtin_ve_vl_pvfmad_vsvvvl",
+    "llvm.ve.vl.pvfmad.vvsvMvl" => "__builtin_ve_vl_pvfmad_vvsvMvl",
+    "llvm.ve.vl.pvfmad.vvsvl" => "__builtin_ve_vl_pvfmad_vvsvl",
+    "llvm.ve.vl.pvfmad.vvsvvl" => "__builtin_ve_vl_pvfmad_vvsvvl",
+    "llvm.ve.vl.pvfmad.vvvvMvl" => "__builtin_ve_vl_pvfmad_vvvvMvl",
+    "llvm.ve.vl.pvfmad.vvvvl" => "__builtin_ve_vl_pvfmad_vvvvl",
+    "llvm.ve.vl.pvfmad.vvvvvl" => "__builtin_ve_vl_pvfmad_vvvvvl",
+    "llvm.ve.vl.pvfmax.vsvMvl" => "__builtin_ve_vl_pvfmax_vsvMvl",
+    "llvm.ve.vl.pvfmax.vsvl" => "__builtin_ve_vl_pvfmax_vsvl",
+    "llvm.ve.vl.pvfmax.vsvvl" => "__builtin_ve_vl_pvfmax_vsvvl",
+    "llvm.ve.vl.pvfmax.vvvMvl" => "__builtin_ve_vl_pvfmax_vvvMvl",
+    "llvm.ve.vl.pvfmax.vvvl" => "__builtin_ve_vl_pvfmax_vvvl",
+    "llvm.ve.vl.pvfmax.vvvvl" => "__builtin_ve_vl_pvfmax_vvvvl",
+    "llvm.ve.vl.pvfmin.vsvMvl" => "__builtin_ve_vl_pvfmin_vsvMvl",
+    "llvm.ve.vl.pvfmin.vsvl" => "__builtin_ve_vl_pvfmin_vsvl",
+    "llvm.ve.vl.pvfmin.vsvvl" => "__builtin_ve_vl_pvfmin_vsvvl",
+    "llvm.ve.vl.pvfmin.vvvMvl" => "__builtin_ve_vl_pvfmin_vvvMvl",
+    "llvm.ve.vl.pvfmin.vvvl" => "__builtin_ve_vl_pvfmin_vvvl",
+    "llvm.ve.vl.pvfmin.vvvvl" => "__builtin_ve_vl_pvfmin_vvvvl",
+    "llvm.ve.vl.pvfmkaf.Ml" => "__builtin_ve_vl_pvfmkaf_Ml",
+    "llvm.ve.vl.pvfmkat.Ml" => "__builtin_ve_vl_pvfmkat_Ml",
+    "llvm.ve.vl.pvfmkseq.MvMl" => "__builtin_ve_vl_pvfmkseq_MvMl",
+    "llvm.ve.vl.pvfmkseq.Mvl" => "__builtin_ve_vl_pvfmkseq_Mvl",
+    "llvm.ve.vl.pvfmkseqnan.MvMl" => "__builtin_ve_vl_pvfmkseqnan_MvMl",
+    "llvm.ve.vl.pvfmkseqnan.Mvl" => "__builtin_ve_vl_pvfmkseqnan_Mvl",
+    "llvm.ve.vl.pvfmksge.MvMl" => "__builtin_ve_vl_pvfmksge_MvMl",
+    "llvm.ve.vl.pvfmksge.Mvl" => "__builtin_ve_vl_pvfmksge_Mvl",
+    "llvm.ve.vl.pvfmksgenan.MvMl" => "__builtin_ve_vl_pvfmksgenan_MvMl",
+    "llvm.ve.vl.pvfmksgenan.Mvl" => "__builtin_ve_vl_pvfmksgenan_Mvl",
+    "llvm.ve.vl.pvfmksgt.MvMl" => "__builtin_ve_vl_pvfmksgt_MvMl",
+    "llvm.ve.vl.pvfmksgt.Mvl" => "__builtin_ve_vl_pvfmksgt_Mvl",
+    "llvm.ve.vl.pvfmksgtnan.MvMl" => "__builtin_ve_vl_pvfmksgtnan_MvMl",
+    "llvm.ve.vl.pvfmksgtnan.Mvl" => "__builtin_ve_vl_pvfmksgtnan_Mvl",
+    "llvm.ve.vl.pvfmksle.MvMl" => "__builtin_ve_vl_pvfmksle_MvMl",
+    "llvm.ve.vl.pvfmksle.Mvl" => "__builtin_ve_vl_pvfmksle_Mvl",
+    "llvm.ve.vl.pvfmkslenan.MvMl" => "__builtin_ve_vl_pvfmkslenan_MvMl",
+    "llvm.ve.vl.pvfmkslenan.Mvl" => "__builtin_ve_vl_pvfmkslenan_Mvl",
+    "llvm.ve.vl.pvfmksloeq.mvl" => "__builtin_ve_vl_pvfmksloeq_mvl",
+    "llvm.ve.vl.pvfmksloeq.mvml" => "__builtin_ve_vl_pvfmksloeq_mvml",
+    "llvm.ve.vl.pvfmksloeqnan.mvl" => "__builtin_ve_vl_pvfmksloeqnan_mvl",
+    "llvm.ve.vl.pvfmksloeqnan.mvml" => "__builtin_ve_vl_pvfmksloeqnan_mvml",
+    "llvm.ve.vl.pvfmksloge.mvl" => "__builtin_ve_vl_pvfmksloge_mvl",
+    "llvm.ve.vl.pvfmksloge.mvml" => "__builtin_ve_vl_pvfmksloge_mvml",
+    "llvm.ve.vl.pvfmkslogenan.mvl" => "__builtin_ve_vl_pvfmkslogenan_mvl",
+    "llvm.ve.vl.pvfmkslogenan.mvml" => "__builtin_ve_vl_pvfmkslogenan_mvml",
+    "llvm.ve.vl.pvfmkslogt.mvl" => "__builtin_ve_vl_pvfmkslogt_mvl",
+    "llvm.ve.vl.pvfmkslogt.mvml" => "__builtin_ve_vl_pvfmkslogt_mvml",
+    "llvm.ve.vl.pvfmkslogtnan.mvl" => "__builtin_ve_vl_pvfmkslogtnan_mvl",
+    "llvm.ve.vl.pvfmkslogtnan.mvml" => "__builtin_ve_vl_pvfmkslogtnan_mvml",
+    "llvm.ve.vl.pvfmkslole.mvl" => "__builtin_ve_vl_pvfmkslole_mvl",
+    "llvm.ve.vl.pvfmkslole.mvml" => "__builtin_ve_vl_pvfmkslole_mvml",
+    "llvm.ve.vl.pvfmkslolenan.mvl" => "__builtin_ve_vl_pvfmkslolenan_mvl",
+    "llvm.ve.vl.pvfmkslolenan.mvml" => "__builtin_ve_vl_pvfmkslolenan_mvml",
+    "llvm.ve.vl.pvfmkslolt.mvl" => "__builtin_ve_vl_pvfmkslolt_mvl",
+    "llvm.ve.vl.pvfmkslolt.mvml" => "__builtin_ve_vl_pvfmkslolt_mvml",
+    "llvm.ve.vl.pvfmksloltnan.mvl" => "__builtin_ve_vl_pvfmksloltnan_mvl",
+    "llvm.ve.vl.pvfmksloltnan.mvml" => "__builtin_ve_vl_pvfmksloltnan_mvml",
+    "llvm.ve.vl.pvfmkslonan.mvl" => "__builtin_ve_vl_pvfmkslonan_mvl",
+    "llvm.ve.vl.pvfmkslonan.mvml" => "__builtin_ve_vl_pvfmkslonan_mvml",
+    "llvm.ve.vl.pvfmkslone.mvl" => "__builtin_ve_vl_pvfmkslone_mvl",
+    "llvm.ve.vl.pvfmkslone.mvml" => "__builtin_ve_vl_pvfmkslone_mvml",
+    "llvm.ve.vl.pvfmkslonenan.mvl" => "__builtin_ve_vl_pvfmkslonenan_mvl",
+    "llvm.ve.vl.pvfmkslonenan.mvml" => "__builtin_ve_vl_pvfmkslonenan_mvml",
+    "llvm.ve.vl.pvfmkslonum.mvl" => "__builtin_ve_vl_pvfmkslonum_mvl",
+    "llvm.ve.vl.pvfmkslonum.mvml" => "__builtin_ve_vl_pvfmkslonum_mvml",
+    "llvm.ve.vl.pvfmkslt.MvMl" => "__builtin_ve_vl_pvfmkslt_MvMl",
+    "llvm.ve.vl.pvfmkslt.Mvl" => "__builtin_ve_vl_pvfmkslt_Mvl",
+    "llvm.ve.vl.pvfmksltnan.MvMl" => "__builtin_ve_vl_pvfmksltnan_MvMl",
+    "llvm.ve.vl.pvfmksltnan.Mvl" => "__builtin_ve_vl_pvfmksltnan_Mvl",
+    "llvm.ve.vl.pvfmksnan.MvMl" => "__builtin_ve_vl_pvfmksnan_MvMl",
+    "llvm.ve.vl.pvfmksnan.Mvl" => "__builtin_ve_vl_pvfmksnan_Mvl",
+    "llvm.ve.vl.pvfmksne.MvMl" => "__builtin_ve_vl_pvfmksne_MvMl",
+    "llvm.ve.vl.pvfmksne.Mvl" => "__builtin_ve_vl_pvfmksne_Mvl",
+    "llvm.ve.vl.pvfmksnenan.MvMl" => "__builtin_ve_vl_pvfmksnenan_MvMl",
+    "llvm.ve.vl.pvfmksnenan.Mvl" => "__builtin_ve_vl_pvfmksnenan_Mvl",
+    "llvm.ve.vl.pvfmksnum.MvMl" => "__builtin_ve_vl_pvfmksnum_MvMl",
+    "llvm.ve.vl.pvfmksnum.Mvl" => "__builtin_ve_vl_pvfmksnum_Mvl",
+    "llvm.ve.vl.pvfmksupeq.mvl" => "__builtin_ve_vl_pvfmksupeq_mvl",
+    "llvm.ve.vl.pvfmksupeq.mvml" => "__builtin_ve_vl_pvfmksupeq_mvml",
+    "llvm.ve.vl.pvfmksupeqnan.mvl" => "__builtin_ve_vl_pvfmksupeqnan_mvl",
+    "llvm.ve.vl.pvfmksupeqnan.mvml" => "__builtin_ve_vl_pvfmksupeqnan_mvml",
+    "llvm.ve.vl.pvfmksupge.mvl" => "__builtin_ve_vl_pvfmksupge_mvl",
+    "llvm.ve.vl.pvfmksupge.mvml" => "__builtin_ve_vl_pvfmksupge_mvml",
+    "llvm.ve.vl.pvfmksupgenan.mvl" => "__builtin_ve_vl_pvfmksupgenan_mvl",
+    "llvm.ve.vl.pvfmksupgenan.mvml" => "__builtin_ve_vl_pvfmksupgenan_mvml",
+    "llvm.ve.vl.pvfmksupgt.mvl" => "__builtin_ve_vl_pvfmksupgt_mvl",
+    "llvm.ve.vl.pvfmksupgt.mvml" => "__builtin_ve_vl_pvfmksupgt_mvml",
+    "llvm.ve.vl.pvfmksupgtnan.mvl" => "__builtin_ve_vl_pvfmksupgtnan_mvl",
+    "llvm.ve.vl.pvfmksupgtnan.mvml" => "__builtin_ve_vl_pvfmksupgtnan_mvml",
+    "llvm.ve.vl.pvfmksuple.mvl" => "__builtin_ve_vl_pvfmksuple_mvl",
+    "llvm.ve.vl.pvfmksuple.mvml" => "__builtin_ve_vl_pvfmksuple_mvml",
+    "llvm.ve.vl.pvfmksuplenan.mvl" => "__builtin_ve_vl_pvfmksuplenan_mvl",
+    "llvm.ve.vl.pvfmksuplenan.mvml" => "__builtin_ve_vl_pvfmksuplenan_mvml",
+    "llvm.ve.vl.pvfmksuplt.mvl" => "__builtin_ve_vl_pvfmksuplt_mvl",
+    "llvm.ve.vl.pvfmksuplt.mvml" => "__builtin_ve_vl_pvfmksuplt_mvml",
+    "llvm.ve.vl.pvfmksupltnan.mvl" => "__builtin_ve_vl_pvfmksupltnan_mvl",
+    "llvm.ve.vl.pvfmksupltnan.mvml" => "__builtin_ve_vl_pvfmksupltnan_mvml",
+    "llvm.ve.vl.pvfmksupnan.mvl" => "__builtin_ve_vl_pvfmksupnan_mvl",
+    "llvm.ve.vl.pvfmksupnan.mvml" => "__builtin_ve_vl_pvfmksupnan_mvml",
+    "llvm.ve.vl.pvfmksupne.mvl" => "__builtin_ve_vl_pvfmksupne_mvl",
+    "llvm.ve.vl.pvfmksupne.mvml" => "__builtin_ve_vl_pvfmksupne_mvml",
+    "llvm.ve.vl.pvfmksupnenan.mvl" => "__builtin_ve_vl_pvfmksupnenan_mvl",
+    "llvm.ve.vl.pvfmksupnenan.mvml" => "__builtin_ve_vl_pvfmksupnenan_mvml",
+    "llvm.ve.vl.pvfmksupnum.mvl" => "__builtin_ve_vl_pvfmksupnum_mvl",
+    "llvm.ve.vl.pvfmksupnum.mvml" => "__builtin_ve_vl_pvfmksupnum_mvml",
+    "llvm.ve.vl.pvfmkweq.MvMl" => "__builtin_ve_vl_pvfmkweq_MvMl",
+    "llvm.ve.vl.pvfmkweq.Mvl" => "__builtin_ve_vl_pvfmkweq_Mvl",
+    "llvm.ve.vl.pvfmkweqnan.MvMl" => "__builtin_ve_vl_pvfmkweqnan_MvMl",
+    "llvm.ve.vl.pvfmkweqnan.Mvl" => "__builtin_ve_vl_pvfmkweqnan_Mvl",
+    "llvm.ve.vl.pvfmkwge.MvMl" => "__builtin_ve_vl_pvfmkwge_MvMl",
+    "llvm.ve.vl.pvfmkwge.Mvl" => "__builtin_ve_vl_pvfmkwge_Mvl",
+    "llvm.ve.vl.pvfmkwgenan.MvMl" => "__builtin_ve_vl_pvfmkwgenan_MvMl",
+    "llvm.ve.vl.pvfmkwgenan.Mvl" => "__builtin_ve_vl_pvfmkwgenan_Mvl",
+    "llvm.ve.vl.pvfmkwgt.MvMl" => "__builtin_ve_vl_pvfmkwgt_MvMl",
+    "llvm.ve.vl.pvfmkwgt.Mvl" => "__builtin_ve_vl_pvfmkwgt_Mvl",
+    "llvm.ve.vl.pvfmkwgtnan.MvMl" => "__builtin_ve_vl_pvfmkwgtnan_MvMl",
+    "llvm.ve.vl.pvfmkwgtnan.Mvl" => "__builtin_ve_vl_pvfmkwgtnan_Mvl",
+    "llvm.ve.vl.pvfmkwle.MvMl" => "__builtin_ve_vl_pvfmkwle_MvMl",
+    "llvm.ve.vl.pvfmkwle.Mvl" => "__builtin_ve_vl_pvfmkwle_Mvl",
+    "llvm.ve.vl.pvfmkwlenan.MvMl" => "__builtin_ve_vl_pvfmkwlenan_MvMl",
+    "llvm.ve.vl.pvfmkwlenan.Mvl" => "__builtin_ve_vl_pvfmkwlenan_Mvl",
+    "llvm.ve.vl.pvfmkwloeq.mvl" => "__builtin_ve_vl_pvfmkwloeq_mvl",
+    "llvm.ve.vl.pvfmkwloeq.mvml" => "__builtin_ve_vl_pvfmkwloeq_mvml",
+    "llvm.ve.vl.pvfmkwloeqnan.mvl" => "__builtin_ve_vl_pvfmkwloeqnan_mvl",
+    "llvm.ve.vl.pvfmkwloeqnan.mvml" => "__builtin_ve_vl_pvfmkwloeqnan_mvml",
+    "llvm.ve.vl.pvfmkwloge.mvl" => "__builtin_ve_vl_pvfmkwloge_mvl",
+    "llvm.ve.vl.pvfmkwloge.mvml" => "__builtin_ve_vl_pvfmkwloge_mvml",
+    "llvm.ve.vl.pvfmkwlogenan.mvl" => "__builtin_ve_vl_pvfmkwlogenan_mvl",
+    "llvm.ve.vl.pvfmkwlogenan.mvml" => "__builtin_ve_vl_pvfmkwlogenan_mvml",
+    "llvm.ve.vl.pvfmkwlogt.mvl" => "__builtin_ve_vl_pvfmkwlogt_mvl",
+    "llvm.ve.vl.pvfmkwlogt.mvml" => "__builtin_ve_vl_pvfmkwlogt_mvml",
+    "llvm.ve.vl.pvfmkwlogtnan.mvl" => "__builtin_ve_vl_pvfmkwlogtnan_mvl",
+    "llvm.ve.vl.pvfmkwlogtnan.mvml" => "__builtin_ve_vl_pvfmkwlogtnan_mvml",
+    "llvm.ve.vl.pvfmkwlole.mvl" => "__builtin_ve_vl_pvfmkwlole_mvl",
+    "llvm.ve.vl.pvfmkwlole.mvml" => "__builtin_ve_vl_pvfmkwlole_mvml",
+    "llvm.ve.vl.pvfmkwlolenan.mvl" => "__builtin_ve_vl_pvfmkwlolenan_mvl",
+    "llvm.ve.vl.pvfmkwlolenan.mvml" => "__builtin_ve_vl_pvfmkwlolenan_mvml",
+    "llvm.ve.vl.pvfmkwlolt.mvl" => "__builtin_ve_vl_pvfmkwlolt_mvl",
+    "llvm.ve.vl.pvfmkwlolt.mvml" => "__builtin_ve_vl_pvfmkwlolt_mvml",
+    "llvm.ve.vl.pvfmkwloltnan.mvl" => "__builtin_ve_vl_pvfmkwloltnan_mvl",
+    "llvm.ve.vl.pvfmkwloltnan.mvml" => "__builtin_ve_vl_pvfmkwloltnan_mvml",
+    "llvm.ve.vl.pvfmkwlonan.mvl" => "__builtin_ve_vl_pvfmkwlonan_mvl",
+    "llvm.ve.vl.pvfmkwlonan.mvml" => "__builtin_ve_vl_pvfmkwlonan_mvml",
+    "llvm.ve.vl.pvfmkwlone.mvl" => "__builtin_ve_vl_pvfmkwlone_mvl",
+    "llvm.ve.vl.pvfmkwlone.mvml" => "__builtin_ve_vl_pvfmkwlone_mvml",
+    "llvm.ve.vl.pvfmkwlonenan.mvl" => "__builtin_ve_vl_pvfmkwlonenan_mvl",
+    "llvm.ve.vl.pvfmkwlonenan.mvml" => "__builtin_ve_vl_pvfmkwlonenan_mvml",
+    "llvm.ve.vl.pvfmkwlonum.mvl" => "__builtin_ve_vl_pvfmkwlonum_mvl",
+    "llvm.ve.vl.pvfmkwlonum.mvml" => "__builtin_ve_vl_pvfmkwlonum_mvml",
+    "llvm.ve.vl.pvfmkwlt.MvMl" => "__builtin_ve_vl_pvfmkwlt_MvMl",
+    "llvm.ve.vl.pvfmkwlt.Mvl" => "__builtin_ve_vl_pvfmkwlt_Mvl",
+    "llvm.ve.vl.pvfmkwltnan.MvMl" => "__builtin_ve_vl_pvfmkwltnan_MvMl",
+    "llvm.ve.vl.pvfmkwltnan.Mvl" => "__builtin_ve_vl_pvfmkwltnan_Mvl",
+    "llvm.ve.vl.pvfmkwnan.MvMl" => "__builtin_ve_vl_pvfmkwnan_MvMl",
+    "llvm.ve.vl.pvfmkwnan.Mvl" => "__builtin_ve_vl_pvfmkwnan_Mvl",
+    "llvm.ve.vl.pvfmkwne.MvMl" => "__builtin_ve_vl_pvfmkwne_MvMl",
+    "llvm.ve.vl.pvfmkwne.Mvl" => "__builtin_ve_vl_pvfmkwne_Mvl",
+    "llvm.ve.vl.pvfmkwnenan.MvMl" => "__builtin_ve_vl_pvfmkwnenan_MvMl",
+    "llvm.ve.vl.pvfmkwnenan.Mvl" => "__builtin_ve_vl_pvfmkwnenan_Mvl",
+    "llvm.ve.vl.pvfmkwnum.MvMl" => "__builtin_ve_vl_pvfmkwnum_MvMl",
+    "llvm.ve.vl.pvfmkwnum.Mvl" => "__builtin_ve_vl_pvfmkwnum_Mvl",
+    "llvm.ve.vl.pvfmkwupeq.mvl" => "__builtin_ve_vl_pvfmkwupeq_mvl",
+    "llvm.ve.vl.pvfmkwupeq.mvml" => "__builtin_ve_vl_pvfmkwupeq_mvml",
+    "llvm.ve.vl.pvfmkwupeqnan.mvl" => "__builtin_ve_vl_pvfmkwupeqnan_mvl",
+    "llvm.ve.vl.pvfmkwupeqnan.mvml" => "__builtin_ve_vl_pvfmkwupeqnan_mvml",
+    "llvm.ve.vl.pvfmkwupge.mvl" => "__builtin_ve_vl_pvfmkwupge_mvl",
+    "llvm.ve.vl.pvfmkwupge.mvml" => "__builtin_ve_vl_pvfmkwupge_mvml",
+    "llvm.ve.vl.pvfmkwupgenan.mvl" => "__builtin_ve_vl_pvfmkwupgenan_mvl",
+    "llvm.ve.vl.pvfmkwupgenan.mvml" => "__builtin_ve_vl_pvfmkwupgenan_mvml",
+    "llvm.ve.vl.pvfmkwupgt.mvl" => "__builtin_ve_vl_pvfmkwupgt_mvl",
+    "llvm.ve.vl.pvfmkwupgt.mvml" => "__builtin_ve_vl_pvfmkwupgt_mvml",
+    "llvm.ve.vl.pvfmkwupgtnan.mvl" => "__builtin_ve_vl_pvfmkwupgtnan_mvl",
+    "llvm.ve.vl.pvfmkwupgtnan.mvml" => "__builtin_ve_vl_pvfmkwupgtnan_mvml",
+    "llvm.ve.vl.pvfmkwuple.mvl" => "__builtin_ve_vl_pvfmkwuple_mvl",
+    "llvm.ve.vl.pvfmkwuple.mvml" => "__builtin_ve_vl_pvfmkwuple_mvml",
+    "llvm.ve.vl.pvfmkwuplenan.mvl" => "__builtin_ve_vl_pvfmkwuplenan_mvl",
+    "llvm.ve.vl.pvfmkwuplenan.mvml" => "__builtin_ve_vl_pvfmkwuplenan_mvml",
+    "llvm.ve.vl.pvfmkwuplt.mvl" => "__builtin_ve_vl_pvfmkwuplt_mvl",
+    "llvm.ve.vl.pvfmkwuplt.mvml" => "__builtin_ve_vl_pvfmkwuplt_mvml",
+    "llvm.ve.vl.pvfmkwupltnan.mvl" => "__builtin_ve_vl_pvfmkwupltnan_mvl",
+    "llvm.ve.vl.pvfmkwupltnan.mvml" => "__builtin_ve_vl_pvfmkwupltnan_mvml",
+    "llvm.ve.vl.pvfmkwupnan.mvl" => "__builtin_ve_vl_pvfmkwupnan_mvl",
+    "llvm.ve.vl.pvfmkwupnan.mvml" => "__builtin_ve_vl_pvfmkwupnan_mvml",
+    "llvm.ve.vl.pvfmkwupne.mvl" => "__builtin_ve_vl_pvfmkwupne_mvl",
+    "llvm.ve.vl.pvfmkwupne.mvml" => "__builtin_ve_vl_pvfmkwupne_mvml",
+    "llvm.ve.vl.pvfmkwupnenan.mvl" => "__builtin_ve_vl_pvfmkwupnenan_mvl",
+    "llvm.ve.vl.pvfmkwupnenan.mvml" => "__builtin_ve_vl_pvfmkwupnenan_mvml",
+    "llvm.ve.vl.pvfmkwupnum.mvl" => "__builtin_ve_vl_pvfmkwupnum_mvl",
+    "llvm.ve.vl.pvfmkwupnum.mvml" => "__builtin_ve_vl_pvfmkwupnum_mvml",
+    "llvm.ve.vl.pvfmsb.vsvvMvl" => "__builtin_ve_vl_pvfmsb_vsvvMvl",
+    "llvm.ve.vl.pvfmsb.vsvvl" => "__builtin_ve_vl_pvfmsb_vsvvl",
+    "llvm.ve.vl.pvfmsb.vsvvvl" => "__builtin_ve_vl_pvfmsb_vsvvvl",
+    "llvm.ve.vl.pvfmsb.vvsvMvl" => "__builtin_ve_vl_pvfmsb_vvsvMvl",
+    "llvm.ve.vl.pvfmsb.vvsvl" => "__builtin_ve_vl_pvfmsb_vvsvl",
+    "llvm.ve.vl.pvfmsb.vvsvvl" => "__builtin_ve_vl_pvfmsb_vvsvvl",
+    "llvm.ve.vl.pvfmsb.vvvvMvl" => "__builtin_ve_vl_pvfmsb_vvvvMvl",
+    "llvm.ve.vl.pvfmsb.vvvvl" => "__builtin_ve_vl_pvfmsb_vvvvl",
+    "llvm.ve.vl.pvfmsb.vvvvvl" => "__builtin_ve_vl_pvfmsb_vvvvvl",
+    "llvm.ve.vl.pvfmul.vsvMvl" => "__builtin_ve_vl_pvfmul_vsvMvl",
+    "llvm.ve.vl.pvfmul.vsvl" => "__builtin_ve_vl_pvfmul_vsvl",
+    "llvm.ve.vl.pvfmul.vsvvl" => "__builtin_ve_vl_pvfmul_vsvvl",
+    "llvm.ve.vl.pvfmul.vvvMvl" => "__builtin_ve_vl_pvfmul_vvvMvl",
+    "llvm.ve.vl.pvfmul.vvvl" => "__builtin_ve_vl_pvfmul_vvvl",
+    "llvm.ve.vl.pvfmul.vvvvl" => "__builtin_ve_vl_pvfmul_vvvvl",
+    "llvm.ve.vl.pvfnmad.vsvvMvl" => "__builtin_ve_vl_pvfnmad_vsvvMvl",
+    "llvm.ve.vl.pvfnmad.vsvvl" => "__builtin_ve_vl_pvfnmad_vsvvl",
+    "llvm.ve.vl.pvfnmad.vsvvvl" => "__builtin_ve_vl_pvfnmad_vsvvvl",
+    "llvm.ve.vl.pvfnmad.vvsvMvl" => "__builtin_ve_vl_pvfnmad_vvsvMvl",
+    "llvm.ve.vl.pvfnmad.vvsvl" => "__builtin_ve_vl_pvfnmad_vvsvl",
+    "llvm.ve.vl.pvfnmad.vvsvvl" => "__builtin_ve_vl_pvfnmad_vvsvvl",
+    "llvm.ve.vl.pvfnmad.vvvvMvl" => "__builtin_ve_vl_pvfnmad_vvvvMvl",
+    "llvm.ve.vl.pvfnmad.vvvvl" => "__builtin_ve_vl_pvfnmad_vvvvl",
+    "llvm.ve.vl.pvfnmad.vvvvvl" => "__builtin_ve_vl_pvfnmad_vvvvvl",
+    "llvm.ve.vl.pvfnmsb.vsvvMvl" => "__builtin_ve_vl_pvfnmsb_vsvvMvl",
+    "llvm.ve.vl.pvfnmsb.vsvvl" => "__builtin_ve_vl_pvfnmsb_vsvvl",
+    "llvm.ve.vl.pvfnmsb.vsvvvl" => "__builtin_ve_vl_pvfnmsb_vsvvvl",
+    "llvm.ve.vl.pvfnmsb.vvsvMvl" => "__builtin_ve_vl_pvfnmsb_vvsvMvl",
+    "llvm.ve.vl.pvfnmsb.vvsvl" => "__builtin_ve_vl_pvfnmsb_vvsvl",
+    "llvm.ve.vl.pvfnmsb.vvsvvl" => "__builtin_ve_vl_pvfnmsb_vvsvvl",
+    "llvm.ve.vl.pvfnmsb.vvvvMvl" => "__builtin_ve_vl_pvfnmsb_vvvvMvl",
+    "llvm.ve.vl.pvfnmsb.vvvvl" => "__builtin_ve_vl_pvfnmsb_vvvvl",
+    "llvm.ve.vl.pvfnmsb.vvvvvl" => "__builtin_ve_vl_pvfnmsb_vvvvvl",
+    "llvm.ve.vl.pvfsub.vsvMvl" => "__builtin_ve_vl_pvfsub_vsvMvl",
+    "llvm.ve.vl.pvfsub.vsvl" => "__builtin_ve_vl_pvfsub_vsvl",
+    "llvm.ve.vl.pvfsub.vsvvl" => "__builtin_ve_vl_pvfsub_vsvvl",
+    "llvm.ve.vl.pvfsub.vvvMvl" => "__builtin_ve_vl_pvfsub_vvvMvl",
+    "llvm.ve.vl.pvfsub.vvvl" => "__builtin_ve_vl_pvfsub_vvvl",
+    "llvm.ve.vl.pvfsub.vvvvl" => "__builtin_ve_vl_pvfsub_vvvvl",
+    "llvm.ve.vl.pvldz.vvMvl" => "__builtin_ve_vl_pvldz_vvMvl",
+    "llvm.ve.vl.pvldz.vvl" => "__builtin_ve_vl_pvldz_vvl",
+    "llvm.ve.vl.pvldz.vvvl" => "__builtin_ve_vl_pvldz_vvvl",
+    "llvm.ve.vl.pvldzlo.vvl" => "__builtin_ve_vl_pvldzlo_vvl",
+    "llvm.ve.vl.pvldzlo.vvmvl" => "__builtin_ve_vl_pvldzlo_vvmvl",
+    "llvm.ve.vl.pvldzlo.vvvl" => "__builtin_ve_vl_pvldzlo_vvvl",
+    "llvm.ve.vl.pvldzup.vvl" => "__builtin_ve_vl_pvldzup_vvl",
+    "llvm.ve.vl.pvldzup.vvmvl" => "__builtin_ve_vl_pvldzup_vvmvl",
+    "llvm.ve.vl.pvldzup.vvvl" => "__builtin_ve_vl_pvldzup_vvvl",
+    "llvm.ve.vl.pvmaxs.vsvMvl" => "__builtin_ve_vl_pvmaxs_vsvMvl",
+    "llvm.ve.vl.pvmaxs.vsvl" => "__builtin_ve_vl_pvmaxs_vsvl",
+    "llvm.ve.vl.pvmaxs.vsvvl" => "__builtin_ve_vl_pvmaxs_vsvvl",
+    "llvm.ve.vl.pvmaxs.vvvMvl" => "__builtin_ve_vl_pvmaxs_vvvMvl",
+    "llvm.ve.vl.pvmaxs.vvvl" => "__builtin_ve_vl_pvmaxs_vvvl",
+    "llvm.ve.vl.pvmaxs.vvvvl" => "__builtin_ve_vl_pvmaxs_vvvvl",
+    "llvm.ve.vl.pvmins.vsvMvl" => "__builtin_ve_vl_pvmins_vsvMvl",
+    "llvm.ve.vl.pvmins.vsvl" => "__builtin_ve_vl_pvmins_vsvl",
+    "llvm.ve.vl.pvmins.vsvvl" => "__builtin_ve_vl_pvmins_vsvvl",
+    "llvm.ve.vl.pvmins.vvvMvl" => "__builtin_ve_vl_pvmins_vvvMvl",
+    "llvm.ve.vl.pvmins.vvvl" => "__builtin_ve_vl_pvmins_vvvl",
+    "llvm.ve.vl.pvmins.vvvvl" => "__builtin_ve_vl_pvmins_vvvvl",
+    "llvm.ve.vl.pvor.vsvMvl" => "__builtin_ve_vl_pvor_vsvMvl",
+    "llvm.ve.vl.pvor.vsvl" => "__builtin_ve_vl_pvor_vsvl",
+    "llvm.ve.vl.pvor.vsvvl" => "__builtin_ve_vl_pvor_vsvvl",
+    "llvm.ve.vl.pvor.vvvMvl" => "__builtin_ve_vl_pvor_vvvMvl",
+    "llvm.ve.vl.pvor.vvvl" => "__builtin_ve_vl_pvor_vvvl",
+    "llvm.ve.vl.pvor.vvvvl" => "__builtin_ve_vl_pvor_vvvvl",
+    "llvm.ve.vl.pvpcnt.vvMvl" => "__builtin_ve_vl_pvpcnt_vvMvl",
+    "llvm.ve.vl.pvpcnt.vvl" => "__builtin_ve_vl_pvpcnt_vvl",
+    "llvm.ve.vl.pvpcnt.vvvl" => "__builtin_ve_vl_pvpcnt_vvvl",
+    "llvm.ve.vl.pvpcntlo.vvl" => "__builtin_ve_vl_pvpcntlo_vvl",
+    "llvm.ve.vl.pvpcntlo.vvmvl" => "__builtin_ve_vl_pvpcntlo_vvmvl",
+    "llvm.ve.vl.pvpcntlo.vvvl" => "__builtin_ve_vl_pvpcntlo_vvvl",
+    "llvm.ve.vl.pvpcntup.vvl" => "__builtin_ve_vl_pvpcntup_vvl",
+    "llvm.ve.vl.pvpcntup.vvmvl" => "__builtin_ve_vl_pvpcntup_vvmvl",
+    "llvm.ve.vl.pvpcntup.vvvl" => "__builtin_ve_vl_pvpcntup_vvvl",
+    "llvm.ve.vl.pvrcp.vvl" => "__builtin_ve_vl_pvrcp_vvl",
+    "llvm.ve.vl.pvrcp.vvvl" => "__builtin_ve_vl_pvrcp_vvvl",
+    "llvm.ve.vl.pvrsqrt.vvl" => "__builtin_ve_vl_pvrsqrt_vvl",
+    "llvm.ve.vl.pvrsqrt.vvvl" => "__builtin_ve_vl_pvrsqrt_vvvl",
+    "llvm.ve.vl.pvrsqrtnex.vvl" => "__builtin_ve_vl_pvrsqrtnex_vvl",
+    "llvm.ve.vl.pvrsqrtnex.vvvl" => "__builtin_ve_vl_pvrsqrtnex_vvvl",
+    "llvm.ve.vl.pvseq.vl" => "__builtin_ve_vl_pvseq_vl",
+    "llvm.ve.vl.pvseq.vvl" => "__builtin_ve_vl_pvseq_vvl",
+    "llvm.ve.vl.pvseqlo.vl" => "__builtin_ve_vl_pvseqlo_vl",
+    "llvm.ve.vl.pvseqlo.vvl" => "__builtin_ve_vl_pvseqlo_vvl",
+    "llvm.ve.vl.pvsequp.vl" => "__builtin_ve_vl_pvsequp_vl",
+    "llvm.ve.vl.pvsequp.vvl" => "__builtin_ve_vl_pvsequp_vvl",
+    "llvm.ve.vl.pvsla.vvsMvl" => "__builtin_ve_vl_pvsla_vvsMvl",
+    "llvm.ve.vl.pvsla.vvsl" => "__builtin_ve_vl_pvsla_vvsl",
+    "llvm.ve.vl.pvsla.vvsvl" => "__builtin_ve_vl_pvsla_vvsvl",
+    "llvm.ve.vl.pvsla.vvvMvl" => "__builtin_ve_vl_pvsla_vvvMvl",
+    "llvm.ve.vl.pvsla.vvvl" => "__builtin_ve_vl_pvsla_vvvl",
+    "llvm.ve.vl.pvsla.vvvvl" => "__builtin_ve_vl_pvsla_vvvvl",
+    "llvm.ve.vl.pvsll.vvsMvl" => "__builtin_ve_vl_pvsll_vvsMvl",
+    "llvm.ve.vl.pvsll.vvsl" => "__builtin_ve_vl_pvsll_vvsl",
+    "llvm.ve.vl.pvsll.vvsvl" => "__builtin_ve_vl_pvsll_vvsvl",
+    "llvm.ve.vl.pvsll.vvvMvl" => "__builtin_ve_vl_pvsll_vvvMvl",
+    "llvm.ve.vl.pvsll.vvvl" => "__builtin_ve_vl_pvsll_vvvl",
+    "llvm.ve.vl.pvsll.vvvvl" => "__builtin_ve_vl_pvsll_vvvvl",
+    "llvm.ve.vl.pvsra.vvsMvl" => "__builtin_ve_vl_pvsra_vvsMvl",
+    "llvm.ve.vl.pvsra.vvsl" => "__builtin_ve_vl_pvsra_vvsl",
+    "llvm.ve.vl.pvsra.vvsvl" => "__builtin_ve_vl_pvsra_vvsvl",
+    "llvm.ve.vl.pvsra.vvvMvl" => "__builtin_ve_vl_pvsra_vvvMvl",
+    "llvm.ve.vl.pvsra.vvvl" => "__builtin_ve_vl_pvsra_vvvl",
+    "llvm.ve.vl.pvsra.vvvvl" => "__builtin_ve_vl_pvsra_vvvvl",
+    "llvm.ve.vl.pvsrl.vvsMvl" => "__builtin_ve_vl_pvsrl_vvsMvl",
+    "llvm.ve.vl.pvsrl.vvsl" => "__builtin_ve_vl_pvsrl_vvsl",
+    "llvm.ve.vl.pvsrl.vvsvl" => "__builtin_ve_vl_pvsrl_vvsvl",
+    "llvm.ve.vl.pvsrl.vvvMvl" => "__builtin_ve_vl_pvsrl_vvvMvl",
+    "llvm.ve.vl.pvsrl.vvvl" => "__builtin_ve_vl_pvsrl_vvvl",
+    "llvm.ve.vl.pvsrl.vvvvl" => "__builtin_ve_vl_pvsrl_vvvvl",
+    "llvm.ve.vl.pvsubs.vsvMvl" => "__builtin_ve_vl_pvsubs_vsvMvl",
+    "llvm.ve.vl.pvsubs.vsvl" => "__builtin_ve_vl_pvsubs_vsvl",
+    "llvm.ve.vl.pvsubs.vsvvl" => "__builtin_ve_vl_pvsubs_vsvvl",
+    "llvm.ve.vl.pvsubs.vvvMvl" => "__builtin_ve_vl_pvsubs_vvvMvl",
+    "llvm.ve.vl.pvsubs.vvvl" => "__builtin_ve_vl_pvsubs_vvvl",
+    "llvm.ve.vl.pvsubs.vvvvl" => "__builtin_ve_vl_pvsubs_vvvvl",
+    "llvm.ve.vl.pvsubu.vsvMvl" => "__builtin_ve_vl_pvsubu_vsvMvl",
+    "llvm.ve.vl.pvsubu.vsvl" => "__builtin_ve_vl_pvsubu_vsvl",
+    "llvm.ve.vl.pvsubu.vsvvl" => "__builtin_ve_vl_pvsubu_vsvvl",
+    "llvm.ve.vl.pvsubu.vvvMvl" => "__builtin_ve_vl_pvsubu_vvvMvl",
+    "llvm.ve.vl.pvsubu.vvvl" => "__builtin_ve_vl_pvsubu_vvvl",
+    "llvm.ve.vl.pvsubu.vvvvl" => "__builtin_ve_vl_pvsubu_vvvvl",
+    "llvm.ve.vl.pvxor.vsvMvl" => "__builtin_ve_vl_pvxor_vsvMvl",
+    "llvm.ve.vl.pvxor.vsvl" => "__builtin_ve_vl_pvxor_vsvl",
+    "llvm.ve.vl.pvxor.vsvvl" => "__builtin_ve_vl_pvxor_vsvvl",
+    "llvm.ve.vl.pvxor.vvvMvl" => "__builtin_ve_vl_pvxor_vvvMvl",
+    "llvm.ve.vl.pvxor.vvvl" => "__builtin_ve_vl_pvxor_vvvl",
+    "llvm.ve.vl.pvxor.vvvvl" => "__builtin_ve_vl_pvxor_vvvvl",
+    "llvm.ve.vl.scr.sss" => "__builtin_ve_vl_scr_sss",
+    "llvm.ve.vl.svm.sMs" => "__builtin_ve_vl_svm_sMs",
+    "llvm.ve.vl.svm.sms" => "__builtin_ve_vl_svm_sms",
+    "llvm.ve.vl.svob" => "__builtin_ve_vl_svob",
+    "llvm.ve.vl.tovm.sml" => "__builtin_ve_vl_tovm_sml",
+    "llvm.ve.vl.tscr.ssss" => "__builtin_ve_vl_tscr_ssss",
+    "llvm.ve.vl.vaddsl.vsvl" => "__builtin_ve_vl_vaddsl_vsvl",
+    "llvm.ve.vl.vaddsl.vsvmvl" => "__builtin_ve_vl_vaddsl_vsvmvl",
+    "llvm.ve.vl.vaddsl.vsvvl" => "__builtin_ve_vl_vaddsl_vsvvl",
+    "llvm.ve.vl.vaddsl.vvvl" => "__builtin_ve_vl_vaddsl_vvvl",
+    "llvm.ve.vl.vaddsl.vvvmvl" => "__builtin_ve_vl_vaddsl_vvvmvl",
+    "llvm.ve.vl.vaddsl.vvvvl" => "__builtin_ve_vl_vaddsl_vvvvl",
+    "llvm.ve.vl.vaddswsx.vsvl" => "__builtin_ve_vl_vaddswsx_vsvl",
+    "llvm.ve.vl.vaddswsx.vsvmvl" => "__builtin_ve_vl_vaddswsx_vsvmvl",
+    "llvm.ve.vl.vaddswsx.vsvvl" => "__builtin_ve_vl_vaddswsx_vsvvl",
+    "llvm.ve.vl.vaddswsx.vvvl" => "__builtin_ve_vl_vaddswsx_vvvl",
+    "llvm.ve.vl.vaddswsx.vvvmvl" => "__builtin_ve_vl_vaddswsx_vvvmvl",
+    "llvm.ve.vl.vaddswsx.vvvvl" => "__builtin_ve_vl_vaddswsx_vvvvl",
+    "llvm.ve.vl.vaddswzx.vsvl" => "__builtin_ve_vl_vaddswzx_vsvl",
+    "llvm.ve.vl.vaddswzx.vsvmvl" => "__builtin_ve_vl_vaddswzx_vsvmvl",
+    "llvm.ve.vl.vaddswzx.vsvvl" => "__builtin_ve_vl_vaddswzx_vsvvl",
+    "llvm.ve.vl.vaddswzx.vvvl" => "__builtin_ve_vl_vaddswzx_vvvl",
+    "llvm.ve.vl.vaddswzx.vvvmvl" => "__builtin_ve_vl_vaddswzx_vvvmvl",
+    "llvm.ve.vl.vaddswzx.vvvvl" => "__builtin_ve_vl_vaddswzx_vvvvl",
+    "llvm.ve.vl.vaddul.vsvl" => "__builtin_ve_vl_vaddul_vsvl",
+    "llvm.ve.vl.vaddul.vsvmvl" => "__builtin_ve_vl_vaddul_vsvmvl",
+    "llvm.ve.vl.vaddul.vsvvl" => "__builtin_ve_vl_vaddul_vsvvl",
+    "llvm.ve.vl.vaddul.vvvl" => "__builtin_ve_vl_vaddul_vvvl",
+    "llvm.ve.vl.vaddul.vvvmvl" => "__builtin_ve_vl_vaddul_vvvmvl",
+    "llvm.ve.vl.vaddul.vvvvl" => "__builtin_ve_vl_vaddul_vvvvl",
+    "llvm.ve.vl.vadduw.vsvl" => "__builtin_ve_vl_vadduw_vsvl",
+    "llvm.ve.vl.vadduw.vsvmvl" => "__builtin_ve_vl_vadduw_vsvmvl",
+    "llvm.ve.vl.vadduw.vsvvl" => "__builtin_ve_vl_vadduw_vsvvl",
+    "llvm.ve.vl.vadduw.vvvl" => "__builtin_ve_vl_vadduw_vvvl",
+    "llvm.ve.vl.vadduw.vvvmvl" => "__builtin_ve_vl_vadduw_vvvmvl",
+    "llvm.ve.vl.vadduw.vvvvl" => "__builtin_ve_vl_vadduw_vvvvl",
+    "llvm.ve.vl.vand.vsvl" => "__builtin_ve_vl_vand_vsvl",
+    "llvm.ve.vl.vand.vsvmvl" => "__builtin_ve_vl_vand_vsvmvl",
+    "llvm.ve.vl.vand.vsvvl" => "__builtin_ve_vl_vand_vsvvl",
+    "llvm.ve.vl.vand.vvvl" => "__builtin_ve_vl_vand_vvvl",
+    "llvm.ve.vl.vand.vvvmvl" => "__builtin_ve_vl_vand_vvvmvl",
+    "llvm.ve.vl.vand.vvvvl" => "__builtin_ve_vl_vand_vvvvl",
+    "llvm.ve.vl.vbrdd.vsl" => "__builtin_ve_vl_vbrdd_vsl",
+    "llvm.ve.vl.vbrdd.vsmvl" => "__builtin_ve_vl_vbrdd_vsmvl",
+    "llvm.ve.vl.vbrdd.vsvl" => "__builtin_ve_vl_vbrdd_vsvl",
+    "llvm.ve.vl.vbrdl.vsl" => "__builtin_ve_vl_vbrdl_vsl",
+    "llvm.ve.vl.vbrdl.vsmvl" => "__builtin_ve_vl_vbrdl_vsmvl",
+    "llvm.ve.vl.vbrdl.vsvl" => "__builtin_ve_vl_vbrdl_vsvl",
+    "llvm.ve.vl.vbrds.vsl" => "__builtin_ve_vl_vbrds_vsl",
+    "llvm.ve.vl.vbrds.vsmvl" => "__builtin_ve_vl_vbrds_vsmvl",
+    "llvm.ve.vl.vbrds.vsvl" => "__builtin_ve_vl_vbrds_vsvl",
+    "llvm.ve.vl.vbrdw.vsl" => "__builtin_ve_vl_vbrdw_vsl",
+    "llvm.ve.vl.vbrdw.vsmvl" => "__builtin_ve_vl_vbrdw_vsmvl",
+    "llvm.ve.vl.vbrdw.vsvl" => "__builtin_ve_vl_vbrdw_vsvl",
+    "llvm.ve.vl.vbrv.vvl" => "__builtin_ve_vl_vbrv_vvl",
+    "llvm.ve.vl.vbrv.vvmvl" => "__builtin_ve_vl_vbrv_vvmvl",
+    "llvm.ve.vl.vbrv.vvvl" => "__builtin_ve_vl_vbrv_vvvl",
+    "llvm.ve.vl.vcmpsl.vsvl" => "__builtin_ve_vl_vcmpsl_vsvl",
+    "llvm.ve.vl.vcmpsl.vsvmvl" => "__builtin_ve_vl_vcmpsl_vsvmvl",
+    "llvm.ve.vl.vcmpsl.vsvvl" => "__builtin_ve_vl_vcmpsl_vsvvl",
+    "llvm.ve.vl.vcmpsl.vvvl" => "__builtin_ve_vl_vcmpsl_vvvl",
+    "llvm.ve.vl.vcmpsl.vvvmvl" => "__builtin_ve_vl_vcmpsl_vvvmvl",
+    "llvm.ve.vl.vcmpsl.vvvvl" => "__builtin_ve_vl_vcmpsl_vvvvl",
+    "llvm.ve.vl.vcmpswsx.vsvl" => "__builtin_ve_vl_vcmpswsx_vsvl",
+    "llvm.ve.vl.vcmpswsx.vsvmvl" => "__builtin_ve_vl_vcmpswsx_vsvmvl",
+    "llvm.ve.vl.vcmpswsx.vsvvl" => "__builtin_ve_vl_vcmpswsx_vsvvl",
+    "llvm.ve.vl.vcmpswsx.vvvl" => "__builtin_ve_vl_vcmpswsx_vvvl",
+    "llvm.ve.vl.vcmpswsx.vvvmvl" => "__builtin_ve_vl_vcmpswsx_vvvmvl",
+    "llvm.ve.vl.vcmpswsx.vvvvl" => "__builtin_ve_vl_vcmpswsx_vvvvl",
+    "llvm.ve.vl.vcmpswzx.vsvl" => "__builtin_ve_vl_vcmpswzx_vsvl",
+    "llvm.ve.vl.vcmpswzx.vsvmvl" => "__builtin_ve_vl_vcmpswzx_vsvmvl",
+    "llvm.ve.vl.vcmpswzx.vsvvl" => "__builtin_ve_vl_vcmpswzx_vsvvl",
+    "llvm.ve.vl.vcmpswzx.vvvl" => "__builtin_ve_vl_vcmpswzx_vvvl",
+    "llvm.ve.vl.vcmpswzx.vvvmvl" => "__builtin_ve_vl_vcmpswzx_vvvmvl",
+    "llvm.ve.vl.vcmpswzx.vvvvl" => "__builtin_ve_vl_vcmpswzx_vvvvl",
+    "llvm.ve.vl.vcmpul.vsvl" => "__builtin_ve_vl_vcmpul_vsvl",
+    "llvm.ve.vl.vcmpul.vsvmvl" => "__builtin_ve_vl_vcmpul_vsvmvl",
+    "llvm.ve.vl.vcmpul.vsvvl" => "__builtin_ve_vl_vcmpul_vsvvl",
+    "llvm.ve.vl.vcmpul.vvvl" => "__builtin_ve_vl_vcmpul_vvvl",
+    "llvm.ve.vl.vcmpul.vvvmvl" => "__builtin_ve_vl_vcmpul_vvvmvl",
+    "llvm.ve.vl.vcmpul.vvvvl" => "__builtin_ve_vl_vcmpul_vvvvl",
+    "llvm.ve.vl.vcmpuw.vsvl" => "__builtin_ve_vl_vcmpuw_vsvl",
+    "llvm.ve.vl.vcmpuw.vsvmvl" => "__builtin_ve_vl_vcmpuw_vsvmvl",
+    "llvm.ve.vl.vcmpuw.vsvvl" => "__builtin_ve_vl_vcmpuw_vsvvl",
+    "llvm.ve.vl.vcmpuw.vvvl" => "__builtin_ve_vl_vcmpuw_vvvl",
+    "llvm.ve.vl.vcmpuw.vvvmvl" => "__builtin_ve_vl_vcmpuw_vvvmvl",
+    "llvm.ve.vl.vcmpuw.vvvvl" => "__builtin_ve_vl_vcmpuw_vvvvl",
+    "llvm.ve.vl.vcp.vvmvl" => "__builtin_ve_vl_vcp_vvmvl",
+    "llvm.ve.vl.vcvtdl.vvl" => "__builtin_ve_vl_vcvtdl_vvl",
+    "llvm.ve.vl.vcvtdl.vvvl" => "__builtin_ve_vl_vcvtdl_vvvl",
+    "llvm.ve.vl.vcvtds.vvl" => "__builtin_ve_vl_vcvtds_vvl",
+    "llvm.ve.vl.vcvtds.vvvl" => "__builtin_ve_vl_vcvtds_vvvl",
+    "llvm.ve.vl.vcvtdw.vvl" => "__builtin_ve_vl_vcvtdw_vvl",
+    "llvm.ve.vl.vcvtdw.vvvl" => "__builtin_ve_vl_vcvtdw_vvvl",
+    "llvm.ve.vl.vcvtld.vvl" => "__builtin_ve_vl_vcvtld_vvl",
+    "llvm.ve.vl.vcvtld.vvmvl" => "__builtin_ve_vl_vcvtld_vvmvl",
+    "llvm.ve.vl.vcvtld.vvvl" => "__builtin_ve_vl_vcvtld_vvvl",
+    "llvm.ve.vl.vcvtldrz.vvl" => "__builtin_ve_vl_vcvtldrz_vvl",
+    "llvm.ve.vl.vcvtldrz.vvmvl" => "__builtin_ve_vl_vcvtldrz_vvmvl",
+    "llvm.ve.vl.vcvtldrz.vvvl" => "__builtin_ve_vl_vcvtldrz_vvvl",
+    "llvm.ve.vl.vcvtsd.vvl" => "__builtin_ve_vl_vcvtsd_vvl",
+    "llvm.ve.vl.vcvtsd.vvvl" => "__builtin_ve_vl_vcvtsd_vvvl",
+    "llvm.ve.vl.vcvtsw.vvl" => "__builtin_ve_vl_vcvtsw_vvl",
+    "llvm.ve.vl.vcvtsw.vvvl" => "__builtin_ve_vl_vcvtsw_vvvl",
+    "llvm.ve.vl.vcvtwdsx.vvl" => "__builtin_ve_vl_vcvtwdsx_vvl",
+    "llvm.ve.vl.vcvtwdsx.vvmvl" => "__builtin_ve_vl_vcvtwdsx_vvmvl",
+    "llvm.ve.vl.vcvtwdsx.vvvl" => "__builtin_ve_vl_vcvtwdsx_vvvl",
+    "llvm.ve.vl.vcvtwdsxrz.vvl" => "__builtin_ve_vl_vcvtwdsxrz_vvl",
+    "llvm.ve.vl.vcvtwdsxrz.vvmvl" => "__builtin_ve_vl_vcvtwdsxrz_vvmvl",
+    "llvm.ve.vl.vcvtwdsxrz.vvvl" => "__builtin_ve_vl_vcvtwdsxrz_vvvl",
+    "llvm.ve.vl.vcvtwdzx.vvl" => "__builtin_ve_vl_vcvtwdzx_vvl",
+    "llvm.ve.vl.vcvtwdzx.vvmvl" => "__builtin_ve_vl_vcvtwdzx_vvmvl",
+    "llvm.ve.vl.vcvtwdzx.vvvl" => "__builtin_ve_vl_vcvtwdzx_vvvl",
+    "llvm.ve.vl.vcvtwdzxrz.vvl" => "__builtin_ve_vl_vcvtwdzxrz_vvl",
+    "llvm.ve.vl.vcvtwdzxrz.vvmvl" => "__builtin_ve_vl_vcvtwdzxrz_vvmvl",
+    "llvm.ve.vl.vcvtwdzxrz.vvvl" => "__builtin_ve_vl_vcvtwdzxrz_vvvl",
+    "llvm.ve.vl.vcvtwssx.vvl" => "__builtin_ve_vl_vcvtwssx_vvl",
+    "llvm.ve.vl.vcvtwssx.vvmvl" => "__builtin_ve_vl_vcvtwssx_vvmvl",
+    "llvm.ve.vl.vcvtwssx.vvvl" => "__builtin_ve_vl_vcvtwssx_vvvl",
+    "llvm.ve.vl.vcvtwssxrz.vvl" => "__builtin_ve_vl_vcvtwssxrz_vvl",
+    "llvm.ve.vl.vcvtwssxrz.vvmvl" => "__builtin_ve_vl_vcvtwssxrz_vvmvl",
+    "llvm.ve.vl.vcvtwssxrz.vvvl" => "__builtin_ve_vl_vcvtwssxrz_vvvl",
+    "llvm.ve.vl.vcvtwszx.vvl" => "__builtin_ve_vl_vcvtwszx_vvl",
+    "llvm.ve.vl.vcvtwszx.vvmvl" => "__builtin_ve_vl_vcvtwszx_vvmvl",
+    "llvm.ve.vl.vcvtwszx.vvvl" => "__builtin_ve_vl_vcvtwszx_vvvl",
+    "llvm.ve.vl.vcvtwszxrz.vvl" => "__builtin_ve_vl_vcvtwszxrz_vvl",
+    "llvm.ve.vl.vcvtwszxrz.vvmvl" => "__builtin_ve_vl_vcvtwszxrz_vvmvl",
+    "llvm.ve.vl.vcvtwszxrz.vvvl" => "__builtin_ve_vl_vcvtwszxrz_vvvl",
+    "llvm.ve.vl.vdivsl.vsvl" => "__builtin_ve_vl_vdivsl_vsvl",
+    "llvm.ve.vl.vdivsl.vsvmvl" => "__builtin_ve_vl_vdivsl_vsvmvl",
+    "llvm.ve.vl.vdivsl.vsvvl" => "__builtin_ve_vl_vdivsl_vsvvl",
+    "llvm.ve.vl.vdivsl.vvsl" => "__builtin_ve_vl_vdivsl_vvsl",
+    "llvm.ve.vl.vdivsl.vvsmvl" => "__builtin_ve_vl_vdivsl_vvsmvl",
+    "llvm.ve.vl.vdivsl.vvsvl" => "__builtin_ve_vl_vdivsl_vvsvl",
+    "llvm.ve.vl.vdivsl.vvvl" => "__builtin_ve_vl_vdivsl_vvvl",
+    "llvm.ve.vl.vdivsl.vvvmvl" => "__builtin_ve_vl_vdivsl_vvvmvl",
+    "llvm.ve.vl.vdivsl.vvvvl" => "__builtin_ve_vl_vdivsl_vvvvl",
+    "llvm.ve.vl.vdivswsx.vsvl" => "__builtin_ve_vl_vdivswsx_vsvl",
+    "llvm.ve.vl.vdivswsx.vsvmvl" => "__builtin_ve_vl_vdivswsx_vsvmvl",
+    "llvm.ve.vl.vdivswsx.vsvvl" => "__builtin_ve_vl_vdivswsx_vsvvl",
+    "llvm.ve.vl.vdivswsx.vvsl" => "__builtin_ve_vl_vdivswsx_vvsl",
+    "llvm.ve.vl.vdivswsx.vvsmvl" => "__builtin_ve_vl_vdivswsx_vvsmvl",
+    "llvm.ve.vl.vdivswsx.vvsvl" => "__builtin_ve_vl_vdivswsx_vvsvl",
+    "llvm.ve.vl.vdivswsx.vvvl" => "__builtin_ve_vl_vdivswsx_vvvl",
+    "llvm.ve.vl.vdivswsx.vvvmvl" => "__builtin_ve_vl_vdivswsx_vvvmvl",
+    "llvm.ve.vl.vdivswsx.vvvvl" => "__builtin_ve_vl_vdivswsx_vvvvl",
+    "llvm.ve.vl.vdivswzx.vsvl" => "__builtin_ve_vl_vdivswzx_vsvl",
+    "llvm.ve.vl.vdivswzx.vsvmvl" => "__builtin_ve_vl_vdivswzx_vsvmvl",
+    "llvm.ve.vl.vdivswzx.vsvvl" => "__builtin_ve_vl_vdivswzx_vsvvl",
+    "llvm.ve.vl.vdivswzx.vvsl" => "__builtin_ve_vl_vdivswzx_vvsl",
+    "llvm.ve.vl.vdivswzx.vvsmvl" => "__builtin_ve_vl_vdivswzx_vvsmvl",
+    "llvm.ve.vl.vdivswzx.vvsvl" => "__builtin_ve_vl_vdivswzx_vvsvl",
+    "llvm.ve.vl.vdivswzx.vvvl" => "__builtin_ve_vl_vdivswzx_vvvl",
+    "llvm.ve.vl.vdivswzx.vvvmvl" => "__builtin_ve_vl_vdivswzx_vvvmvl",
+    "llvm.ve.vl.vdivswzx.vvvvl" => "__builtin_ve_vl_vdivswzx_vvvvl",
+    "llvm.ve.vl.vdivul.vsvl" => "__builtin_ve_vl_vdivul_vsvl",
+    "llvm.ve.vl.vdivul.vsvmvl" => "__builtin_ve_vl_vdivul_vsvmvl",
+    "llvm.ve.vl.vdivul.vsvvl" => "__builtin_ve_vl_vdivul_vsvvl",
+    "llvm.ve.vl.vdivul.vvsl" => "__builtin_ve_vl_vdivul_vvsl",
+    "llvm.ve.vl.vdivul.vvsmvl" => "__builtin_ve_vl_vdivul_vvsmvl",
+    "llvm.ve.vl.vdivul.vvsvl" => "__builtin_ve_vl_vdivul_vvsvl",
+    "llvm.ve.vl.vdivul.vvvl" => "__builtin_ve_vl_vdivul_vvvl",
+    "llvm.ve.vl.vdivul.vvvmvl" => "__builtin_ve_vl_vdivul_vvvmvl",
+    "llvm.ve.vl.vdivul.vvvvl" => "__builtin_ve_vl_vdivul_vvvvl",
+    "llvm.ve.vl.vdivuw.vsvl" => "__builtin_ve_vl_vdivuw_vsvl",
+    "llvm.ve.vl.vdivuw.vsvmvl" => "__builtin_ve_vl_vdivuw_vsvmvl",
+    "llvm.ve.vl.vdivuw.vsvvl" => "__builtin_ve_vl_vdivuw_vsvvl",
+    "llvm.ve.vl.vdivuw.vvsl" => "__builtin_ve_vl_vdivuw_vvsl",
+    "llvm.ve.vl.vdivuw.vvsmvl" => "__builtin_ve_vl_vdivuw_vvsmvl",
+    "llvm.ve.vl.vdivuw.vvsvl" => "__builtin_ve_vl_vdivuw_vvsvl",
+    "llvm.ve.vl.vdivuw.vvvl" => "__builtin_ve_vl_vdivuw_vvvl",
+    "llvm.ve.vl.vdivuw.vvvmvl" => "__builtin_ve_vl_vdivuw_vvvmvl",
+    "llvm.ve.vl.vdivuw.vvvvl" => "__builtin_ve_vl_vdivuw_vvvvl",
+    "llvm.ve.vl.veqv.vsvl" => "__builtin_ve_vl_veqv_vsvl",
+    "llvm.ve.vl.veqv.vsvmvl" => "__builtin_ve_vl_veqv_vsvmvl",
+    "llvm.ve.vl.veqv.vsvvl" => "__builtin_ve_vl_veqv_vsvvl",
+    "llvm.ve.vl.veqv.vvvl" => "__builtin_ve_vl_veqv_vvvl",
+    "llvm.ve.vl.veqv.vvvmvl" => "__builtin_ve_vl_veqv_vvvmvl",
+    "llvm.ve.vl.veqv.vvvvl" => "__builtin_ve_vl_veqv_vvvvl",
+    "llvm.ve.vl.vex.vvmvl" => "__builtin_ve_vl_vex_vvmvl",
+    "llvm.ve.vl.vfaddd.vsvl" => "__builtin_ve_vl_vfaddd_vsvl",
+    "llvm.ve.vl.vfaddd.vsvmvl" => "__builtin_ve_vl_vfaddd_vsvmvl",
+    "llvm.ve.vl.vfaddd.vsvvl" => "__builtin_ve_vl_vfaddd_vsvvl",
+    "llvm.ve.vl.vfaddd.vvvl" => "__builtin_ve_vl_vfaddd_vvvl",
+    "llvm.ve.vl.vfaddd.vvvmvl" => "__builtin_ve_vl_vfaddd_vvvmvl",
+    "llvm.ve.vl.vfaddd.vvvvl" => "__builtin_ve_vl_vfaddd_vvvvl",
+    "llvm.ve.vl.vfadds.vsvl" => "__builtin_ve_vl_vfadds_vsvl",
+    "llvm.ve.vl.vfadds.vsvmvl" => "__builtin_ve_vl_vfadds_vsvmvl",
+    "llvm.ve.vl.vfadds.vsvvl" => "__builtin_ve_vl_vfadds_vsvvl",
+    "llvm.ve.vl.vfadds.vvvl" => "__builtin_ve_vl_vfadds_vvvl",
+    "llvm.ve.vl.vfadds.vvvmvl" => "__builtin_ve_vl_vfadds_vvvmvl",
+    "llvm.ve.vl.vfadds.vvvvl" => "__builtin_ve_vl_vfadds_vvvvl",
+    "llvm.ve.vl.vfcmpd.vsvl" => "__builtin_ve_vl_vfcmpd_vsvl",
+    "llvm.ve.vl.vfcmpd.vsvmvl" => "__builtin_ve_vl_vfcmpd_vsvmvl",
+    "llvm.ve.vl.vfcmpd.vsvvl" => "__builtin_ve_vl_vfcmpd_vsvvl",
+    "llvm.ve.vl.vfcmpd.vvvl" => "__builtin_ve_vl_vfcmpd_vvvl",
+    "llvm.ve.vl.vfcmpd.vvvmvl" => "__builtin_ve_vl_vfcmpd_vvvmvl",
+    "llvm.ve.vl.vfcmpd.vvvvl" => "__builtin_ve_vl_vfcmpd_vvvvl",
+    "llvm.ve.vl.vfcmps.vsvl" => "__builtin_ve_vl_vfcmps_vsvl",
+    "llvm.ve.vl.vfcmps.vsvmvl" => "__builtin_ve_vl_vfcmps_vsvmvl",
+    "llvm.ve.vl.vfcmps.vsvvl" => "__builtin_ve_vl_vfcmps_vsvvl",
+    "llvm.ve.vl.vfcmps.vvvl" => "__builtin_ve_vl_vfcmps_vvvl",
+    "llvm.ve.vl.vfcmps.vvvmvl" => "__builtin_ve_vl_vfcmps_vvvmvl",
+    "llvm.ve.vl.vfcmps.vvvvl" => "__builtin_ve_vl_vfcmps_vvvvl",
+    "llvm.ve.vl.vfdivd.vsvl" => "__builtin_ve_vl_vfdivd_vsvl",
+    "llvm.ve.vl.vfdivd.vsvmvl" => "__builtin_ve_vl_vfdivd_vsvmvl",
+    "llvm.ve.vl.vfdivd.vsvvl" => "__builtin_ve_vl_vfdivd_vsvvl",
+    "llvm.ve.vl.vfdivd.vvvl" => "__builtin_ve_vl_vfdivd_vvvl",
+    "llvm.ve.vl.vfdivd.vvvmvl" => "__builtin_ve_vl_vfdivd_vvvmvl",
+    "llvm.ve.vl.vfdivd.vvvvl" => "__builtin_ve_vl_vfdivd_vvvvl",
+    "llvm.ve.vl.vfdivs.vsvl" => "__builtin_ve_vl_vfdivs_vsvl",
+    "llvm.ve.vl.vfdivs.vsvmvl" => "__builtin_ve_vl_vfdivs_vsvmvl",
+    "llvm.ve.vl.vfdivs.vsvvl" => "__builtin_ve_vl_vfdivs_vsvvl",
+    "llvm.ve.vl.vfdivs.vvvl" => "__builtin_ve_vl_vfdivs_vvvl",
+    "llvm.ve.vl.vfdivs.vvvmvl" => "__builtin_ve_vl_vfdivs_vvvmvl",
+    "llvm.ve.vl.vfdivs.vvvvl" => "__builtin_ve_vl_vfdivs_vvvvl",
+    "llvm.ve.vl.vfmadd.vsvvl" => "__builtin_ve_vl_vfmadd_vsvvl",
+    "llvm.ve.vl.vfmadd.vsvvmvl" => "__builtin_ve_vl_vfmadd_vsvvmvl",
+    "llvm.ve.vl.vfmadd.vsvvvl" => "__builtin_ve_vl_vfmadd_vsvvvl",
+    "llvm.ve.vl.vfmadd.vvsvl" => "__builtin_ve_vl_vfmadd_vvsvl",
+    "llvm.ve.vl.vfmadd.vvsvmvl" => "__builtin_ve_vl_vfmadd_vvsvmvl",
+    "llvm.ve.vl.vfmadd.vvsvvl" => "__builtin_ve_vl_vfmadd_vvsvvl",
+    "llvm.ve.vl.vfmadd.vvvvl" => "__builtin_ve_vl_vfmadd_vvvvl",
+    "llvm.ve.vl.vfmadd.vvvvmvl" => "__builtin_ve_vl_vfmadd_vvvvmvl",
+    "llvm.ve.vl.vfmadd.vvvvvl" => "__builtin_ve_vl_vfmadd_vvvvvl",
+    "llvm.ve.vl.vfmads.vsvvl" => "__builtin_ve_vl_vfmads_vsvvl",
+    "llvm.ve.vl.vfmads.vsvvmvl" => "__builtin_ve_vl_vfmads_vsvvmvl",
+    "llvm.ve.vl.vfmads.vsvvvl" => "__builtin_ve_vl_vfmads_vsvvvl",
+    "llvm.ve.vl.vfmads.vvsvl" => "__builtin_ve_vl_vfmads_vvsvl",
+    "llvm.ve.vl.vfmads.vvsvmvl" => "__builtin_ve_vl_vfmads_vvsvmvl",
+    "llvm.ve.vl.vfmads.vvsvvl" => "__builtin_ve_vl_vfmads_vvsvvl",
+    "llvm.ve.vl.vfmads.vvvvl" => "__builtin_ve_vl_vfmads_vvvvl",
+    "llvm.ve.vl.vfmads.vvvvmvl" => "__builtin_ve_vl_vfmads_vvvvmvl",
+    "llvm.ve.vl.vfmads.vvvvvl" => "__builtin_ve_vl_vfmads_vvvvvl",
+    "llvm.ve.vl.vfmaxd.vsvl" => "__builtin_ve_vl_vfmaxd_vsvl",
+    "llvm.ve.vl.vfmaxd.vsvmvl" => "__builtin_ve_vl_vfmaxd_vsvmvl",
+    "llvm.ve.vl.vfmaxd.vsvvl" => "__builtin_ve_vl_vfmaxd_vsvvl",
+    "llvm.ve.vl.vfmaxd.vvvl" => "__builtin_ve_vl_vfmaxd_vvvl",
+    "llvm.ve.vl.vfmaxd.vvvmvl" => "__builtin_ve_vl_vfmaxd_vvvmvl",
+    "llvm.ve.vl.vfmaxd.vvvvl" => "__builtin_ve_vl_vfmaxd_vvvvl",
+    "llvm.ve.vl.vfmaxs.vsvl" => "__builtin_ve_vl_vfmaxs_vsvl",
+    "llvm.ve.vl.vfmaxs.vsvmvl" => "__builtin_ve_vl_vfmaxs_vsvmvl",
+    "llvm.ve.vl.vfmaxs.vsvvl" => "__builtin_ve_vl_vfmaxs_vsvvl",
+    "llvm.ve.vl.vfmaxs.vvvl" => "__builtin_ve_vl_vfmaxs_vvvl",
+    "llvm.ve.vl.vfmaxs.vvvmvl" => "__builtin_ve_vl_vfmaxs_vvvmvl",
+    "llvm.ve.vl.vfmaxs.vvvvl" => "__builtin_ve_vl_vfmaxs_vvvvl",
+    "llvm.ve.vl.vfmind.vsvl" => "__builtin_ve_vl_vfmind_vsvl",
+    "llvm.ve.vl.vfmind.vsvmvl" => "__builtin_ve_vl_vfmind_vsvmvl",
+    "llvm.ve.vl.vfmind.vsvvl" => "__builtin_ve_vl_vfmind_vsvvl",
+    "llvm.ve.vl.vfmind.vvvl" => "__builtin_ve_vl_vfmind_vvvl",
+    "llvm.ve.vl.vfmind.vvvmvl" => "__builtin_ve_vl_vfmind_vvvmvl",
+    "llvm.ve.vl.vfmind.vvvvl" => "__builtin_ve_vl_vfmind_vvvvl",
+    "llvm.ve.vl.vfmins.vsvl" => "__builtin_ve_vl_vfmins_vsvl",
+    "llvm.ve.vl.vfmins.vsvmvl" => "__builtin_ve_vl_vfmins_vsvmvl",
+    "llvm.ve.vl.vfmins.vsvvl" => "__builtin_ve_vl_vfmins_vsvvl",
+    "llvm.ve.vl.vfmins.vvvl" => "__builtin_ve_vl_vfmins_vvvl",
+    "llvm.ve.vl.vfmins.vvvmvl" => "__builtin_ve_vl_vfmins_vvvmvl",
+    "llvm.ve.vl.vfmins.vvvvl" => "__builtin_ve_vl_vfmins_vvvvl",
+    "llvm.ve.vl.vfmkdeq.mvl" => "__builtin_ve_vl_vfmkdeq_mvl",
+    "llvm.ve.vl.vfmkdeq.mvml" => "__builtin_ve_vl_vfmkdeq_mvml",
+    "llvm.ve.vl.vfmkdeqnan.mvl" => "__builtin_ve_vl_vfmkdeqnan_mvl",
+    "llvm.ve.vl.vfmkdeqnan.mvml" => "__builtin_ve_vl_vfmkdeqnan_mvml",
+    "llvm.ve.vl.vfmkdge.mvl" => "__builtin_ve_vl_vfmkdge_mvl",
+    "llvm.ve.vl.vfmkdge.mvml" => "__builtin_ve_vl_vfmkdge_mvml",
+    "llvm.ve.vl.vfmkdgenan.mvl" => "__builtin_ve_vl_vfmkdgenan_mvl",
+    "llvm.ve.vl.vfmkdgenan.mvml" => "__builtin_ve_vl_vfmkdgenan_mvml",
+    "llvm.ve.vl.vfmkdgt.mvl" => "__builtin_ve_vl_vfmkdgt_mvl",
+    "llvm.ve.vl.vfmkdgt.mvml" => "__builtin_ve_vl_vfmkdgt_mvml",
+    "llvm.ve.vl.vfmkdgtnan.mvl" => "__builtin_ve_vl_vfmkdgtnan_mvl",
+    "llvm.ve.vl.vfmkdgtnan.mvml" => "__builtin_ve_vl_vfmkdgtnan_mvml",
+    "llvm.ve.vl.vfmkdle.mvl" => "__builtin_ve_vl_vfmkdle_mvl",
+    "llvm.ve.vl.vfmkdle.mvml" => "__builtin_ve_vl_vfmkdle_mvml",
+    "llvm.ve.vl.vfmkdlenan.mvl" => "__builtin_ve_vl_vfmkdlenan_mvl",
+    "llvm.ve.vl.vfmkdlenan.mvml" => "__builtin_ve_vl_vfmkdlenan_mvml",
+    "llvm.ve.vl.vfmkdlt.mvl" => "__builtin_ve_vl_vfmkdlt_mvl",
+    "llvm.ve.vl.vfmkdlt.mvml" => "__builtin_ve_vl_vfmkdlt_mvml",
+    "llvm.ve.vl.vfmkdltnan.mvl" => "__builtin_ve_vl_vfmkdltnan_mvl",
+    "llvm.ve.vl.vfmkdltnan.mvml" => "__builtin_ve_vl_vfmkdltnan_mvml",
+    "llvm.ve.vl.vfmkdnan.mvl" => "__builtin_ve_vl_vfmkdnan_mvl",
+    "llvm.ve.vl.vfmkdnan.mvml" => "__builtin_ve_vl_vfmkdnan_mvml",
+    "llvm.ve.vl.vfmkdne.mvl" => "__builtin_ve_vl_vfmkdne_mvl",
+    "llvm.ve.vl.vfmkdne.mvml" => "__builtin_ve_vl_vfmkdne_mvml",
+    "llvm.ve.vl.vfmkdnenan.mvl" => "__builtin_ve_vl_vfmkdnenan_mvl",
+    "llvm.ve.vl.vfmkdnenan.mvml" => "__builtin_ve_vl_vfmkdnenan_mvml",
+    "llvm.ve.vl.vfmkdnum.mvl" => "__builtin_ve_vl_vfmkdnum_mvl",
+    "llvm.ve.vl.vfmkdnum.mvml" => "__builtin_ve_vl_vfmkdnum_mvml",
+    "llvm.ve.vl.vfmklaf.ml" => "__builtin_ve_vl_vfmklaf_ml",
+    "llvm.ve.vl.vfmklat.ml" => "__builtin_ve_vl_vfmklat_ml",
+    "llvm.ve.vl.vfmkleq.mvl" => "__builtin_ve_vl_vfmkleq_mvl",
+    "llvm.ve.vl.vfmkleq.mvml" => "__builtin_ve_vl_vfmkleq_mvml",
+    "llvm.ve.vl.vfmkleqnan.mvl" => "__builtin_ve_vl_vfmkleqnan_mvl",
+    "llvm.ve.vl.vfmkleqnan.mvml" => "__builtin_ve_vl_vfmkleqnan_mvml",
+    "llvm.ve.vl.vfmklge.mvl" => "__builtin_ve_vl_vfmklge_mvl",
+    "llvm.ve.vl.vfmklge.mvml" => "__builtin_ve_vl_vfmklge_mvml",
+    "llvm.ve.vl.vfmklgenan.mvl" => "__builtin_ve_vl_vfmklgenan_mvl",
+    "llvm.ve.vl.vfmklgenan.mvml" => "__builtin_ve_vl_vfmklgenan_mvml",
+    "llvm.ve.vl.vfmklgt.mvl" => "__builtin_ve_vl_vfmklgt_mvl",
+    "llvm.ve.vl.vfmklgt.mvml" => "__builtin_ve_vl_vfmklgt_mvml",
+    "llvm.ve.vl.vfmklgtnan.mvl" => "__builtin_ve_vl_vfmklgtnan_mvl",
+    "llvm.ve.vl.vfmklgtnan.mvml" => "__builtin_ve_vl_vfmklgtnan_mvml",
+    "llvm.ve.vl.vfmklle.mvl" => "__builtin_ve_vl_vfmklle_mvl",
+    "llvm.ve.vl.vfmklle.mvml" => "__builtin_ve_vl_vfmklle_mvml",
+    "llvm.ve.vl.vfmkllenan.mvl" => "__builtin_ve_vl_vfmkllenan_mvl",
+    "llvm.ve.vl.vfmkllenan.mvml" => "__builtin_ve_vl_vfmkllenan_mvml",
+    "llvm.ve.vl.vfmkllt.mvl" => "__builtin_ve_vl_vfmkllt_mvl",
+    "llvm.ve.vl.vfmkllt.mvml" => "__builtin_ve_vl_vfmkllt_mvml",
+    "llvm.ve.vl.vfmklltnan.mvl" => "__builtin_ve_vl_vfmklltnan_mvl",
+    "llvm.ve.vl.vfmklltnan.mvml" => "__builtin_ve_vl_vfmklltnan_mvml",
+    "llvm.ve.vl.vfmklnan.mvl" => "__builtin_ve_vl_vfmklnan_mvl",
+    "llvm.ve.vl.vfmklnan.mvml" => "__builtin_ve_vl_vfmklnan_mvml",
+    "llvm.ve.vl.vfmklne.mvl" => "__builtin_ve_vl_vfmklne_mvl",
+    "llvm.ve.vl.vfmklne.mvml" => "__builtin_ve_vl_vfmklne_mvml",
+    "llvm.ve.vl.vfmklnenan.mvl" => "__builtin_ve_vl_vfmklnenan_mvl",
+    "llvm.ve.vl.vfmklnenan.mvml" => "__builtin_ve_vl_vfmklnenan_mvml",
+    "llvm.ve.vl.vfmklnum.mvl" => "__builtin_ve_vl_vfmklnum_mvl",
+    "llvm.ve.vl.vfmklnum.mvml" => "__builtin_ve_vl_vfmklnum_mvml",
+    "llvm.ve.vl.vfmkseq.mvl" => "__builtin_ve_vl_vfmkseq_mvl",
+    "llvm.ve.vl.vfmkseq.mvml" => "__builtin_ve_vl_vfmkseq_mvml",
+    "llvm.ve.vl.vfmkseqnan.mvl" => "__builtin_ve_vl_vfmkseqnan_mvl",
+    "llvm.ve.vl.vfmkseqnan.mvml" => "__builtin_ve_vl_vfmkseqnan_mvml",
+    "llvm.ve.vl.vfmksge.mvl" => "__builtin_ve_vl_vfmksge_mvl",
+    "llvm.ve.vl.vfmksge.mvml" => "__builtin_ve_vl_vfmksge_mvml",
+    "llvm.ve.vl.vfmksgenan.mvl" => "__builtin_ve_vl_vfmksgenan_mvl",
+    "llvm.ve.vl.vfmksgenan.mvml" => "__builtin_ve_vl_vfmksgenan_mvml",
+    "llvm.ve.vl.vfmksgt.mvl" => "__builtin_ve_vl_vfmksgt_mvl",
+    "llvm.ve.vl.vfmksgt.mvml" => "__builtin_ve_vl_vfmksgt_mvml",
+    "llvm.ve.vl.vfmksgtnan.mvl" => "__builtin_ve_vl_vfmksgtnan_mvl",
+    "llvm.ve.vl.vfmksgtnan.mvml" => "__builtin_ve_vl_vfmksgtnan_mvml",
+    "llvm.ve.vl.vfmksle.mvl" => "__builtin_ve_vl_vfmksle_mvl",
+    "llvm.ve.vl.vfmksle.mvml" => "__builtin_ve_vl_vfmksle_mvml",
+    "llvm.ve.vl.vfmkslenan.mvl" => "__builtin_ve_vl_vfmkslenan_mvl",
+    "llvm.ve.vl.vfmkslenan.mvml" => "__builtin_ve_vl_vfmkslenan_mvml",
+    "llvm.ve.vl.vfmkslt.mvl" => "__builtin_ve_vl_vfmkslt_mvl",
+    "llvm.ve.vl.vfmkslt.mvml" => "__builtin_ve_vl_vfmkslt_mvml",
+    "llvm.ve.vl.vfmksltnan.mvl" => "__builtin_ve_vl_vfmksltnan_mvl",
+    "llvm.ve.vl.vfmksltnan.mvml" => "__builtin_ve_vl_vfmksltnan_mvml",
+    "llvm.ve.vl.vfmksnan.mvl" => "__builtin_ve_vl_vfmksnan_mvl",
+    "llvm.ve.vl.vfmksnan.mvml" => "__builtin_ve_vl_vfmksnan_mvml",
+    "llvm.ve.vl.vfmksne.mvl" => "__builtin_ve_vl_vfmksne_mvl",
+    "llvm.ve.vl.vfmksne.mvml" => "__builtin_ve_vl_vfmksne_mvml",
+    "llvm.ve.vl.vfmksnenan.mvl" => "__builtin_ve_vl_vfmksnenan_mvl",
+    "llvm.ve.vl.vfmksnenan.mvml" => "__builtin_ve_vl_vfmksnenan_mvml",
+    "llvm.ve.vl.vfmksnum.mvl" => "__builtin_ve_vl_vfmksnum_mvl",
+    "llvm.ve.vl.vfmksnum.mvml" => "__builtin_ve_vl_vfmksnum_mvml",
+    "llvm.ve.vl.vfmkweq.mvl" => "__builtin_ve_vl_vfmkweq_mvl",
+    "llvm.ve.vl.vfmkweq.mvml" => "__builtin_ve_vl_vfmkweq_mvml",
+    "llvm.ve.vl.vfmkweqnan.mvl" => "__builtin_ve_vl_vfmkweqnan_mvl",
+    "llvm.ve.vl.vfmkweqnan.mvml" => "__builtin_ve_vl_vfmkweqnan_mvml",
+    "llvm.ve.vl.vfmkwge.mvl" => "__builtin_ve_vl_vfmkwge_mvl",
+    "llvm.ve.vl.vfmkwge.mvml" => "__builtin_ve_vl_vfmkwge_mvml",
+    "llvm.ve.vl.vfmkwgenan.mvl" => "__builtin_ve_vl_vfmkwgenan_mvl",
+    "llvm.ve.vl.vfmkwgenan.mvml" => "__builtin_ve_vl_vfmkwgenan_mvml",
+    "llvm.ve.vl.vfmkwgt.mvl" => "__builtin_ve_vl_vfmkwgt_mvl",
+    "llvm.ve.vl.vfmkwgt.mvml" => "__builtin_ve_vl_vfmkwgt_mvml",
+    "llvm.ve.vl.vfmkwgtnan.mvl" => "__builtin_ve_vl_vfmkwgtnan_mvl",
+    "llvm.ve.vl.vfmkwgtnan.mvml" => "__builtin_ve_vl_vfmkwgtnan_mvml",
+    "llvm.ve.vl.vfmkwle.mvl" => "__builtin_ve_vl_vfmkwle_mvl",
+    "llvm.ve.vl.vfmkwle.mvml" => "__builtin_ve_vl_vfmkwle_mvml",
+    "llvm.ve.vl.vfmkwlenan.mvl" => "__builtin_ve_vl_vfmkwlenan_mvl",
+    "llvm.ve.vl.vfmkwlenan.mvml" => "__builtin_ve_vl_vfmkwlenan_mvml",
+    "llvm.ve.vl.vfmkwlt.mvl" => "__builtin_ve_vl_vfmkwlt_mvl",
+    "llvm.ve.vl.vfmkwlt.mvml" => "__builtin_ve_vl_vfmkwlt_mvml",
+    "llvm.ve.vl.vfmkwltnan.mvl" => "__builtin_ve_vl_vfmkwltnan_mvl",
+    "llvm.ve.vl.vfmkwltnan.mvml" => "__builtin_ve_vl_vfmkwltnan_mvml",
+    "llvm.ve.vl.vfmkwnan.mvl" => "__builtin_ve_vl_vfmkwnan_mvl",
+    "llvm.ve.vl.vfmkwnan.mvml" => "__builtin_ve_vl_vfmkwnan_mvml",
+    "llvm.ve.vl.vfmkwne.mvl" => "__builtin_ve_vl_vfmkwne_mvl",
+    "llvm.ve.vl.vfmkwne.mvml" => "__builtin_ve_vl_vfmkwne_mvml",
+    "llvm.ve.vl.vfmkwnenan.mvl" => "__builtin_ve_vl_vfmkwnenan_mvl",
+    "llvm.ve.vl.vfmkwnenan.mvml" => "__builtin_ve_vl_vfmkwnenan_mvml",
+    "llvm.ve.vl.vfmkwnum.mvl" => "__builtin_ve_vl_vfmkwnum_mvl",
+    "llvm.ve.vl.vfmkwnum.mvml" => "__builtin_ve_vl_vfmkwnum_mvml",
+    "llvm.ve.vl.vfmsbd.vsvvl" => "__builtin_ve_vl_vfmsbd_vsvvl",
+    "llvm.ve.vl.vfmsbd.vsvvmvl" => "__builtin_ve_vl_vfmsbd_vsvvmvl",
+    "llvm.ve.vl.vfmsbd.vsvvvl" => "__builtin_ve_vl_vfmsbd_vsvvvl",
+    "llvm.ve.vl.vfmsbd.vvsvl" => "__builtin_ve_vl_vfmsbd_vvsvl",
+    "llvm.ve.vl.vfmsbd.vvsvmvl" => "__builtin_ve_vl_vfmsbd_vvsvmvl",
+    "llvm.ve.vl.vfmsbd.vvsvvl" => "__builtin_ve_vl_vfmsbd_vvsvvl",
+    "llvm.ve.vl.vfmsbd.vvvvl" => "__builtin_ve_vl_vfmsbd_vvvvl",
+    "llvm.ve.vl.vfmsbd.vvvvmvl" => "__builtin_ve_vl_vfmsbd_vvvvmvl",
+    "llvm.ve.vl.vfmsbd.vvvvvl" => "__builtin_ve_vl_vfmsbd_vvvvvl",
+    "llvm.ve.vl.vfmsbs.vsvvl" => "__builtin_ve_vl_vfmsbs_vsvvl",
+    "llvm.ve.vl.vfmsbs.vsvvmvl" => "__builtin_ve_vl_vfmsbs_vsvvmvl",
+    "llvm.ve.vl.vfmsbs.vsvvvl" => "__builtin_ve_vl_vfmsbs_vsvvvl",
+    "llvm.ve.vl.vfmsbs.vvsvl" => "__builtin_ve_vl_vfmsbs_vvsvl",
+    "llvm.ve.vl.vfmsbs.vvsvmvl" => "__builtin_ve_vl_vfmsbs_vvsvmvl",
+    "llvm.ve.vl.vfmsbs.vvsvvl" => "__builtin_ve_vl_vfmsbs_vvsvvl",
+    "llvm.ve.vl.vfmsbs.vvvvl" => "__builtin_ve_vl_vfmsbs_vvvvl",
+    "llvm.ve.vl.vfmsbs.vvvvmvl" => "__builtin_ve_vl_vfmsbs_vvvvmvl",
+    "llvm.ve.vl.vfmsbs.vvvvvl" => "__builtin_ve_vl_vfmsbs_vvvvvl",
+    "llvm.ve.vl.vfmuld.vsvl" => "__builtin_ve_vl_vfmuld_vsvl",
+    "llvm.ve.vl.vfmuld.vsvmvl" => "__builtin_ve_vl_vfmuld_vsvmvl",
+    "llvm.ve.vl.vfmuld.vsvvl" => "__builtin_ve_vl_vfmuld_vsvvl",
+    "llvm.ve.vl.vfmuld.vvvl" => "__builtin_ve_vl_vfmuld_vvvl",
+    "llvm.ve.vl.vfmuld.vvvmvl" => "__builtin_ve_vl_vfmuld_vvvmvl",
+    "llvm.ve.vl.vfmuld.vvvvl" => "__builtin_ve_vl_vfmuld_vvvvl",
+    "llvm.ve.vl.vfmuls.vsvl" => "__builtin_ve_vl_vfmuls_vsvl",
+    "llvm.ve.vl.vfmuls.vsvmvl" => "__builtin_ve_vl_vfmuls_vsvmvl",
+    "llvm.ve.vl.vfmuls.vsvvl" => "__builtin_ve_vl_vfmuls_vsvvl",
+    "llvm.ve.vl.vfmuls.vvvl" => "__builtin_ve_vl_vfmuls_vvvl",
+    "llvm.ve.vl.vfmuls.vvvmvl" => "__builtin_ve_vl_vfmuls_vvvmvl",
+    "llvm.ve.vl.vfmuls.vvvvl" => "__builtin_ve_vl_vfmuls_vvvvl",
+    "llvm.ve.vl.vfnmadd.vsvvl" => "__builtin_ve_vl_vfnmadd_vsvvl",
+    "llvm.ve.vl.vfnmadd.vsvvmvl" => "__builtin_ve_vl_vfnmadd_vsvvmvl",
+    "llvm.ve.vl.vfnmadd.vsvvvl" => "__builtin_ve_vl_vfnmadd_vsvvvl",
+    "llvm.ve.vl.vfnmadd.vvsvl" => "__builtin_ve_vl_vfnmadd_vvsvl",
+    "llvm.ve.vl.vfnmadd.vvsvmvl" => "__builtin_ve_vl_vfnmadd_vvsvmvl",
+    "llvm.ve.vl.vfnmadd.vvsvvl" => "__builtin_ve_vl_vfnmadd_vvsvvl",
+    "llvm.ve.vl.vfnmadd.vvvvl" => "__builtin_ve_vl_vfnmadd_vvvvl",
+    "llvm.ve.vl.vfnmadd.vvvvmvl" => "__builtin_ve_vl_vfnmadd_vvvvmvl",
+    "llvm.ve.vl.vfnmadd.vvvvvl" => "__builtin_ve_vl_vfnmadd_vvvvvl",
+    "llvm.ve.vl.vfnmads.vsvvl" => "__builtin_ve_vl_vfnmads_vsvvl",
+    "llvm.ve.vl.vfnmads.vsvvmvl" => "__builtin_ve_vl_vfnmads_vsvvmvl",
+    "llvm.ve.vl.vfnmads.vsvvvl" => "__builtin_ve_vl_vfnmads_vsvvvl",
+    "llvm.ve.vl.vfnmads.vvsvl" => "__builtin_ve_vl_vfnmads_vvsvl",
+    "llvm.ve.vl.vfnmads.vvsvmvl" => "__builtin_ve_vl_vfnmads_vvsvmvl",
+    "llvm.ve.vl.vfnmads.vvsvvl" => "__builtin_ve_vl_vfnmads_vvsvvl",
+    "llvm.ve.vl.vfnmads.vvvvl" => "__builtin_ve_vl_vfnmads_vvvvl",
+    "llvm.ve.vl.vfnmads.vvvvmvl" => "__builtin_ve_vl_vfnmads_vvvvmvl",
+    "llvm.ve.vl.vfnmads.vvvvvl" => "__builtin_ve_vl_vfnmads_vvvvvl",
+    "llvm.ve.vl.vfnmsbd.vsvvl" => "__builtin_ve_vl_vfnmsbd_vsvvl",
+    "llvm.ve.vl.vfnmsbd.vsvvmvl" => "__builtin_ve_vl_vfnmsbd_vsvvmvl",
+    "llvm.ve.vl.vfnmsbd.vsvvvl" => "__builtin_ve_vl_vfnmsbd_vsvvvl",
+    "llvm.ve.vl.vfnmsbd.vvsvl" => "__builtin_ve_vl_vfnmsbd_vvsvl",
+    "llvm.ve.vl.vfnmsbd.vvsvmvl" => "__builtin_ve_vl_vfnmsbd_vvsvmvl",
+    "llvm.ve.vl.vfnmsbd.vvsvvl" => "__builtin_ve_vl_vfnmsbd_vvsvvl",
+    "llvm.ve.vl.vfnmsbd.vvvvl" => "__builtin_ve_vl_vfnmsbd_vvvvl",
+    "llvm.ve.vl.vfnmsbd.vvvvmvl" => "__builtin_ve_vl_vfnmsbd_vvvvmvl",
+    "llvm.ve.vl.vfnmsbd.vvvvvl" => "__builtin_ve_vl_vfnmsbd_vvvvvl",
+    "llvm.ve.vl.vfnmsbs.vsvvl" => "__builtin_ve_vl_vfnmsbs_vsvvl",
+    "llvm.ve.vl.vfnmsbs.vsvvmvl" => "__builtin_ve_vl_vfnmsbs_vsvvmvl",
+    "llvm.ve.vl.vfnmsbs.vsvvvl" => "__builtin_ve_vl_vfnmsbs_vsvvvl",
+    "llvm.ve.vl.vfnmsbs.vvsvl" => "__builtin_ve_vl_vfnmsbs_vvsvl",
+    "llvm.ve.vl.vfnmsbs.vvsvmvl" => "__builtin_ve_vl_vfnmsbs_vvsvmvl",
+    "llvm.ve.vl.vfnmsbs.vvsvvl" => "__builtin_ve_vl_vfnmsbs_vvsvvl",
+    "llvm.ve.vl.vfnmsbs.vvvvl" => "__builtin_ve_vl_vfnmsbs_vvvvl",
+    "llvm.ve.vl.vfnmsbs.vvvvmvl" => "__builtin_ve_vl_vfnmsbs_vvvvmvl",
+    "llvm.ve.vl.vfnmsbs.vvvvvl" => "__builtin_ve_vl_vfnmsbs_vvvvvl",
+    "llvm.ve.vl.vfrmaxdfst.vvl" => "__builtin_ve_vl_vfrmaxdfst_vvl",
+    "llvm.ve.vl.vfrmaxdfst.vvvl" => "__builtin_ve_vl_vfrmaxdfst_vvvl",
+    "llvm.ve.vl.vfrmaxdlst.vvl" => "__builtin_ve_vl_vfrmaxdlst_vvl",
+    "llvm.ve.vl.vfrmaxdlst.vvvl" => "__builtin_ve_vl_vfrmaxdlst_vvvl",
+    "llvm.ve.vl.vfrmaxsfst.vvl" => "__builtin_ve_vl_vfrmaxsfst_vvl",
+    "llvm.ve.vl.vfrmaxsfst.vvvl" => "__builtin_ve_vl_vfrmaxsfst_vvvl",
+    "llvm.ve.vl.vfrmaxslst.vvl" => "__builtin_ve_vl_vfrmaxslst_vvl",
+    "llvm.ve.vl.vfrmaxslst.vvvl" => "__builtin_ve_vl_vfrmaxslst_vvvl",
+    "llvm.ve.vl.vfrmindfst.vvl" => "__builtin_ve_vl_vfrmindfst_vvl",
+    "llvm.ve.vl.vfrmindfst.vvvl" => "__builtin_ve_vl_vfrmindfst_vvvl",
+    "llvm.ve.vl.vfrmindlst.vvl" => "__builtin_ve_vl_vfrmindlst_vvl",
+    "llvm.ve.vl.vfrmindlst.vvvl" => "__builtin_ve_vl_vfrmindlst_vvvl",
+    "llvm.ve.vl.vfrminsfst.vvl" => "__builtin_ve_vl_vfrminsfst_vvl",
+    "llvm.ve.vl.vfrminsfst.vvvl" => "__builtin_ve_vl_vfrminsfst_vvvl",
+    "llvm.ve.vl.vfrminslst.vvl" => "__builtin_ve_vl_vfrminslst_vvl",
+    "llvm.ve.vl.vfrminslst.vvvl" => "__builtin_ve_vl_vfrminslst_vvvl",
+    "llvm.ve.vl.vfsqrtd.vvl" => "__builtin_ve_vl_vfsqrtd_vvl",
+    "llvm.ve.vl.vfsqrtd.vvvl" => "__builtin_ve_vl_vfsqrtd_vvvl",
+    "llvm.ve.vl.vfsqrts.vvl" => "__builtin_ve_vl_vfsqrts_vvl",
+    "llvm.ve.vl.vfsqrts.vvvl" => "__builtin_ve_vl_vfsqrts_vvvl",
+    "llvm.ve.vl.vfsubd.vsvl" => "__builtin_ve_vl_vfsubd_vsvl",
+    "llvm.ve.vl.vfsubd.vsvmvl" => "__builtin_ve_vl_vfsubd_vsvmvl",
+    "llvm.ve.vl.vfsubd.vsvvl" => "__builtin_ve_vl_vfsubd_vsvvl",
+    "llvm.ve.vl.vfsubd.vvvl" => "__builtin_ve_vl_vfsubd_vvvl",
+    "llvm.ve.vl.vfsubd.vvvmvl" => "__builtin_ve_vl_vfsubd_vvvmvl",
+    "llvm.ve.vl.vfsubd.vvvvl" => "__builtin_ve_vl_vfsubd_vvvvl",
+    "llvm.ve.vl.vfsubs.vsvl" => "__builtin_ve_vl_vfsubs_vsvl",
+    "llvm.ve.vl.vfsubs.vsvmvl" => "__builtin_ve_vl_vfsubs_vsvmvl",
+    "llvm.ve.vl.vfsubs.vsvvl" => "__builtin_ve_vl_vfsubs_vsvvl",
+    "llvm.ve.vl.vfsubs.vvvl" => "__builtin_ve_vl_vfsubs_vvvl",
+    "llvm.ve.vl.vfsubs.vvvmvl" => "__builtin_ve_vl_vfsubs_vvvmvl",
+    "llvm.ve.vl.vfsubs.vvvvl" => "__builtin_ve_vl_vfsubs_vvvvl",
+    "llvm.ve.vl.vfsumd.vvl" => "__builtin_ve_vl_vfsumd_vvl",
+    "llvm.ve.vl.vfsumd.vvml" => "__builtin_ve_vl_vfsumd_vvml",
+    "llvm.ve.vl.vfsums.vvl" => "__builtin_ve_vl_vfsums_vvl",
+    "llvm.ve.vl.vfsums.vvml" => "__builtin_ve_vl_vfsums_vvml",
+    "llvm.ve.vl.vgt.vvssl" => "__builtin_ve_vl_vgt_vvssl",
+    "llvm.ve.vl.vgt.vvssml" => "__builtin_ve_vl_vgt_vvssml",
+    "llvm.ve.vl.vgt.vvssmvl" => "__builtin_ve_vl_vgt_vvssmvl",
+    "llvm.ve.vl.vgt.vvssvl" => "__builtin_ve_vl_vgt_vvssvl",
+    "llvm.ve.vl.vgtlsx.vvssl" => "__builtin_ve_vl_vgtlsx_vvssl",
+    "llvm.ve.vl.vgtlsx.vvssml" => "__builtin_ve_vl_vgtlsx_vvssml",
+    "llvm.ve.vl.vgtlsx.vvssmvl" => "__builtin_ve_vl_vgtlsx_vvssmvl",
+    "llvm.ve.vl.vgtlsx.vvssvl" => "__builtin_ve_vl_vgtlsx_vvssvl",
+    "llvm.ve.vl.vgtlsxnc.vvssl" => "__builtin_ve_vl_vgtlsxnc_vvssl",
+    "llvm.ve.vl.vgtlsxnc.vvssml" => "__builtin_ve_vl_vgtlsxnc_vvssml",
+    "llvm.ve.vl.vgtlsxnc.vvssmvl" => "__builtin_ve_vl_vgtlsxnc_vvssmvl",
+    "llvm.ve.vl.vgtlsxnc.vvssvl" => "__builtin_ve_vl_vgtlsxnc_vvssvl",
+    "llvm.ve.vl.vgtlzx.vvssl" => "__builtin_ve_vl_vgtlzx_vvssl",
+    "llvm.ve.vl.vgtlzx.vvssml" => "__builtin_ve_vl_vgtlzx_vvssml",
+    "llvm.ve.vl.vgtlzx.vvssmvl" => "__builtin_ve_vl_vgtlzx_vvssmvl",
+    "llvm.ve.vl.vgtlzx.vvssvl" => "__builtin_ve_vl_vgtlzx_vvssvl",
+    "llvm.ve.vl.vgtlzxnc.vvssl" => "__builtin_ve_vl_vgtlzxnc_vvssl",
+    "llvm.ve.vl.vgtlzxnc.vvssml" => "__builtin_ve_vl_vgtlzxnc_vvssml",
+    "llvm.ve.vl.vgtlzxnc.vvssmvl" => "__builtin_ve_vl_vgtlzxnc_vvssmvl",
+    "llvm.ve.vl.vgtlzxnc.vvssvl" => "__builtin_ve_vl_vgtlzxnc_vvssvl",
+    "llvm.ve.vl.vgtnc.vvssl" => "__builtin_ve_vl_vgtnc_vvssl",
+    "llvm.ve.vl.vgtnc.vvssml" => "__builtin_ve_vl_vgtnc_vvssml",
+    "llvm.ve.vl.vgtnc.vvssmvl" => "__builtin_ve_vl_vgtnc_vvssmvl",
+    "llvm.ve.vl.vgtnc.vvssvl" => "__builtin_ve_vl_vgtnc_vvssvl",
+    "llvm.ve.vl.vgtu.vvssl" => "__builtin_ve_vl_vgtu_vvssl",
+    "llvm.ve.vl.vgtu.vvssml" => "__builtin_ve_vl_vgtu_vvssml",
+    "llvm.ve.vl.vgtu.vvssmvl" => "__builtin_ve_vl_vgtu_vvssmvl",
+    "llvm.ve.vl.vgtu.vvssvl" => "__builtin_ve_vl_vgtu_vvssvl",
+    "llvm.ve.vl.vgtunc.vvssl" => "__builtin_ve_vl_vgtunc_vvssl",
+    "llvm.ve.vl.vgtunc.vvssml" => "__builtin_ve_vl_vgtunc_vvssml",
+    "llvm.ve.vl.vgtunc.vvssmvl" => "__builtin_ve_vl_vgtunc_vvssmvl",
+    "llvm.ve.vl.vgtunc.vvssvl" => "__builtin_ve_vl_vgtunc_vvssvl",
+    "llvm.ve.vl.vld.vssl" => "__builtin_ve_vl_vld_vssl",
+    "llvm.ve.vl.vld.vssvl" => "__builtin_ve_vl_vld_vssvl",
+    "llvm.ve.vl.vld2d.vssl" => "__builtin_ve_vl_vld2d_vssl",
+    "llvm.ve.vl.vld2d.vssvl" => "__builtin_ve_vl_vld2d_vssvl",
+    "llvm.ve.vl.vld2dnc.vssl" => "__builtin_ve_vl_vld2dnc_vssl",
+    "llvm.ve.vl.vld2dnc.vssvl" => "__builtin_ve_vl_vld2dnc_vssvl",
+    "llvm.ve.vl.vldl2dsx.vssl" => "__builtin_ve_vl_vldl2dsx_vssl",
+    "llvm.ve.vl.vldl2dsx.vssvl" => "__builtin_ve_vl_vldl2dsx_vssvl",
+    "llvm.ve.vl.vldl2dsxnc.vssl" => "__builtin_ve_vl_vldl2dsxnc_vssl",
+    "llvm.ve.vl.vldl2dsxnc.vssvl" => "__builtin_ve_vl_vldl2dsxnc_vssvl",
+    "llvm.ve.vl.vldl2dzx.vssl" => "__builtin_ve_vl_vldl2dzx_vssl",
+    "llvm.ve.vl.vldl2dzx.vssvl" => "__builtin_ve_vl_vldl2dzx_vssvl",
+    "llvm.ve.vl.vldl2dzxnc.vssl" => "__builtin_ve_vl_vldl2dzxnc_vssl",
+    "llvm.ve.vl.vldl2dzxnc.vssvl" => "__builtin_ve_vl_vldl2dzxnc_vssvl",
+    "llvm.ve.vl.vldlsx.vssl" => "__builtin_ve_vl_vldlsx_vssl",
+    "llvm.ve.vl.vldlsx.vssvl" => "__builtin_ve_vl_vldlsx_vssvl",
+    "llvm.ve.vl.vldlsxnc.vssl" => "__builtin_ve_vl_vldlsxnc_vssl",
+    "llvm.ve.vl.vldlsxnc.vssvl" => "__builtin_ve_vl_vldlsxnc_vssvl",
+    "llvm.ve.vl.vldlzx.vssl" => "__builtin_ve_vl_vldlzx_vssl",
+    "llvm.ve.vl.vldlzx.vssvl" => "__builtin_ve_vl_vldlzx_vssvl",
+    "llvm.ve.vl.vldlzxnc.vssl" => "__builtin_ve_vl_vldlzxnc_vssl",
+    "llvm.ve.vl.vldlzxnc.vssvl" => "__builtin_ve_vl_vldlzxnc_vssvl",
+    "llvm.ve.vl.vldnc.vssl" => "__builtin_ve_vl_vldnc_vssl",
+    "llvm.ve.vl.vldnc.vssvl" => "__builtin_ve_vl_vldnc_vssvl",
+    "llvm.ve.vl.vldu.vssl" => "__builtin_ve_vl_vldu_vssl",
+    "llvm.ve.vl.vldu.vssvl" => "__builtin_ve_vl_vldu_vssvl",
+    "llvm.ve.vl.vldu2d.vssl" => "__builtin_ve_vl_vldu2d_vssl",
+    "llvm.ve.vl.vldu2d.vssvl" => "__builtin_ve_vl_vldu2d_vssvl",
+    "llvm.ve.vl.vldu2dnc.vssl" => "__builtin_ve_vl_vldu2dnc_vssl",
+    "llvm.ve.vl.vldu2dnc.vssvl" => "__builtin_ve_vl_vldu2dnc_vssvl",
+    "llvm.ve.vl.vldunc.vssl" => "__builtin_ve_vl_vldunc_vssl",
+    "llvm.ve.vl.vldunc.vssvl" => "__builtin_ve_vl_vldunc_vssvl",
+    "llvm.ve.vl.vldz.vvl" => "__builtin_ve_vl_vldz_vvl",
+    "llvm.ve.vl.vldz.vvmvl" => "__builtin_ve_vl_vldz_vvmvl",
+    "llvm.ve.vl.vldz.vvvl" => "__builtin_ve_vl_vldz_vvvl",
+    "llvm.ve.vl.vmaxsl.vsvl" => "__builtin_ve_vl_vmaxsl_vsvl",
+    "llvm.ve.vl.vmaxsl.vsvmvl" => "__builtin_ve_vl_vmaxsl_vsvmvl",
+    "llvm.ve.vl.vmaxsl.vsvvl" => "__builtin_ve_vl_vmaxsl_vsvvl",
+    "llvm.ve.vl.vmaxsl.vvvl" => "__builtin_ve_vl_vmaxsl_vvvl",
+    "llvm.ve.vl.vmaxsl.vvvmvl" => "__builtin_ve_vl_vmaxsl_vvvmvl",
+    "llvm.ve.vl.vmaxsl.vvvvl" => "__builtin_ve_vl_vmaxsl_vvvvl",
+    "llvm.ve.vl.vmaxswsx.vsvl" => "__builtin_ve_vl_vmaxswsx_vsvl",
+    "llvm.ve.vl.vmaxswsx.vsvmvl" => "__builtin_ve_vl_vmaxswsx_vsvmvl",
+    "llvm.ve.vl.vmaxswsx.vsvvl" => "__builtin_ve_vl_vmaxswsx_vsvvl",
+    "llvm.ve.vl.vmaxswsx.vvvl" => "__builtin_ve_vl_vmaxswsx_vvvl",
+    "llvm.ve.vl.vmaxswsx.vvvmvl" => "__builtin_ve_vl_vmaxswsx_vvvmvl",
+    "llvm.ve.vl.vmaxswsx.vvvvl" => "__builtin_ve_vl_vmaxswsx_vvvvl",
+    "llvm.ve.vl.vmaxswzx.vsvl" => "__builtin_ve_vl_vmaxswzx_vsvl",
+    "llvm.ve.vl.vmaxswzx.vsvmvl" => "__builtin_ve_vl_vmaxswzx_vsvmvl",
+    "llvm.ve.vl.vmaxswzx.vsvvl" => "__builtin_ve_vl_vmaxswzx_vsvvl",
+    "llvm.ve.vl.vmaxswzx.vvvl" => "__builtin_ve_vl_vmaxswzx_vvvl",
+    "llvm.ve.vl.vmaxswzx.vvvmvl" => "__builtin_ve_vl_vmaxswzx_vvvmvl",
+    "llvm.ve.vl.vmaxswzx.vvvvl" => "__builtin_ve_vl_vmaxswzx_vvvvl",
+    "llvm.ve.vl.vminsl.vsvl" => "__builtin_ve_vl_vminsl_vsvl",
+    "llvm.ve.vl.vminsl.vsvmvl" => "__builtin_ve_vl_vminsl_vsvmvl",
+    "llvm.ve.vl.vminsl.vsvvl" => "__builtin_ve_vl_vminsl_vsvvl",
+    "llvm.ve.vl.vminsl.vvvl" => "__builtin_ve_vl_vminsl_vvvl",
+    "llvm.ve.vl.vminsl.vvvmvl" => "__builtin_ve_vl_vminsl_vvvmvl",
+    "llvm.ve.vl.vminsl.vvvvl" => "__builtin_ve_vl_vminsl_vvvvl",
+    "llvm.ve.vl.vminswsx.vsvl" => "__builtin_ve_vl_vminswsx_vsvl",
+    "llvm.ve.vl.vminswsx.vsvmvl" => "__builtin_ve_vl_vminswsx_vsvmvl",
+    "llvm.ve.vl.vminswsx.vsvvl" => "__builtin_ve_vl_vminswsx_vsvvl",
+    "llvm.ve.vl.vminswsx.vvvl" => "__builtin_ve_vl_vminswsx_vvvl",
+    "llvm.ve.vl.vminswsx.vvvmvl" => "__builtin_ve_vl_vminswsx_vvvmvl",
+    "llvm.ve.vl.vminswsx.vvvvl" => "__builtin_ve_vl_vminswsx_vvvvl",
+    "llvm.ve.vl.vminswzx.vsvl" => "__builtin_ve_vl_vminswzx_vsvl",
+    "llvm.ve.vl.vminswzx.vsvmvl" => "__builtin_ve_vl_vminswzx_vsvmvl",
+    "llvm.ve.vl.vminswzx.vsvvl" => "__builtin_ve_vl_vminswzx_vsvvl",
+    "llvm.ve.vl.vminswzx.vvvl" => "__builtin_ve_vl_vminswzx_vvvl",
+    "llvm.ve.vl.vminswzx.vvvmvl" => "__builtin_ve_vl_vminswzx_vvvmvl",
+    "llvm.ve.vl.vminswzx.vvvvl" => "__builtin_ve_vl_vminswzx_vvvvl",
+    "llvm.ve.vl.vmrg.vsvml" => "__builtin_ve_vl_vmrg_vsvml",
+    "llvm.ve.vl.vmrg.vsvmvl" => "__builtin_ve_vl_vmrg_vsvmvl",
+    "llvm.ve.vl.vmrg.vvvml" => "__builtin_ve_vl_vmrg_vvvml",
+    "llvm.ve.vl.vmrg.vvvmvl" => "__builtin_ve_vl_vmrg_vvvmvl",
+    "llvm.ve.vl.vmrgw.vsvMl" => "__builtin_ve_vl_vmrgw_vsvMl",
+    "llvm.ve.vl.vmrgw.vsvMvl" => "__builtin_ve_vl_vmrgw_vsvMvl",
+    "llvm.ve.vl.vmrgw.vvvMl" => "__builtin_ve_vl_vmrgw_vvvMl",
+    "llvm.ve.vl.vmrgw.vvvMvl" => "__builtin_ve_vl_vmrgw_vvvMvl",
+    "llvm.ve.vl.vmulsl.vsvl" => "__builtin_ve_vl_vmulsl_vsvl",
+    "llvm.ve.vl.vmulsl.vsvmvl" => "__builtin_ve_vl_vmulsl_vsvmvl",
+    "llvm.ve.vl.vmulsl.vsvvl" => "__builtin_ve_vl_vmulsl_vsvvl",
+    "llvm.ve.vl.vmulsl.vvvl" => "__builtin_ve_vl_vmulsl_vvvl",
+    "llvm.ve.vl.vmulsl.vvvmvl" => "__builtin_ve_vl_vmulsl_vvvmvl",
+    "llvm.ve.vl.vmulsl.vvvvl" => "__builtin_ve_vl_vmulsl_vvvvl",
+    "llvm.ve.vl.vmulslw.vsvl" => "__builtin_ve_vl_vmulslw_vsvl",
+    "llvm.ve.vl.vmulslw.vsvvl" => "__builtin_ve_vl_vmulslw_vsvvl",
+    "llvm.ve.vl.vmulslw.vvvl" => "__builtin_ve_vl_vmulslw_vvvl",
+    "llvm.ve.vl.vmulslw.vvvvl" => "__builtin_ve_vl_vmulslw_vvvvl",
+    "llvm.ve.vl.vmulswsx.vsvl" => "__builtin_ve_vl_vmulswsx_vsvl",
+    "llvm.ve.vl.vmulswsx.vsvmvl" => "__builtin_ve_vl_vmulswsx_vsvmvl",
+    "llvm.ve.vl.vmulswsx.vsvvl" => "__builtin_ve_vl_vmulswsx_vsvvl",
+    "llvm.ve.vl.vmulswsx.vvvl" => "__builtin_ve_vl_vmulswsx_vvvl",
+    "llvm.ve.vl.vmulswsx.vvvmvl" => "__builtin_ve_vl_vmulswsx_vvvmvl",
+    "llvm.ve.vl.vmulswsx.vvvvl" => "__builtin_ve_vl_vmulswsx_vvvvl",
+    "llvm.ve.vl.vmulswzx.vsvl" => "__builtin_ve_vl_vmulswzx_vsvl",
+    "llvm.ve.vl.vmulswzx.vsvmvl" => "__builtin_ve_vl_vmulswzx_vsvmvl",
+    "llvm.ve.vl.vmulswzx.vsvvl" => "__builtin_ve_vl_vmulswzx_vsvvl",
+    "llvm.ve.vl.vmulswzx.vvvl" => "__builtin_ve_vl_vmulswzx_vvvl",
+    "llvm.ve.vl.vmulswzx.vvvmvl" => "__builtin_ve_vl_vmulswzx_vvvmvl",
+    "llvm.ve.vl.vmulswzx.vvvvl" => "__builtin_ve_vl_vmulswzx_vvvvl",
+    "llvm.ve.vl.vmulul.vsvl" => "__builtin_ve_vl_vmulul_vsvl",
+    "llvm.ve.vl.vmulul.vsvmvl" => "__builtin_ve_vl_vmulul_vsvmvl",
+    "llvm.ve.vl.vmulul.vsvvl" => "__builtin_ve_vl_vmulul_vsvvl",
+    "llvm.ve.vl.vmulul.vvvl" => "__builtin_ve_vl_vmulul_vvvl",
+    "llvm.ve.vl.vmulul.vvvmvl" => "__builtin_ve_vl_vmulul_vvvmvl",
+    "llvm.ve.vl.vmulul.vvvvl" => "__builtin_ve_vl_vmulul_vvvvl",
+    "llvm.ve.vl.vmuluw.vsvl" => "__builtin_ve_vl_vmuluw_vsvl",
+    "llvm.ve.vl.vmuluw.vsvmvl" => "__builtin_ve_vl_vmuluw_vsvmvl",
+    "llvm.ve.vl.vmuluw.vsvvl" => "__builtin_ve_vl_vmuluw_vsvvl",
+    "llvm.ve.vl.vmuluw.vvvl" => "__builtin_ve_vl_vmuluw_vvvl",
+    "llvm.ve.vl.vmuluw.vvvmvl" => "__builtin_ve_vl_vmuluw_vvvmvl",
+    "llvm.ve.vl.vmuluw.vvvvl" => "__builtin_ve_vl_vmuluw_vvvvl",
+    "llvm.ve.vl.vmv.vsvl" => "__builtin_ve_vl_vmv_vsvl",
+    "llvm.ve.vl.vmv.vsvmvl" => "__builtin_ve_vl_vmv_vsvmvl",
+    "llvm.ve.vl.vmv.vsvvl" => "__builtin_ve_vl_vmv_vsvvl",
+    "llvm.ve.vl.vor.vsvl" => "__builtin_ve_vl_vor_vsvl",
+    "llvm.ve.vl.vor.vsvmvl" => "__builtin_ve_vl_vor_vsvmvl",
+    "llvm.ve.vl.vor.vsvvl" => "__builtin_ve_vl_vor_vsvvl",
+    "llvm.ve.vl.vor.vvvl" => "__builtin_ve_vl_vor_vvvl",
+    "llvm.ve.vl.vor.vvvmvl" => "__builtin_ve_vl_vor_vvvmvl",
+    "llvm.ve.vl.vor.vvvvl" => "__builtin_ve_vl_vor_vvvvl",
+    "llvm.ve.vl.vpcnt.vvl" => "__builtin_ve_vl_vpcnt_vvl",
+    "llvm.ve.vl.vpcnt.vvmvl" => "__builtin_ve_vl_vpcnt_vvmvl",
+    "llvm.ve.vl.vpcnt.vvvl" => "__builtin_ve_vl_vpcnt_vvvl",
+    "llvm.ve.vl.vrand.vvl" => "__builtin_ve_vl_vrand_vvl",
+    "llvm.ve.vl.vrand.vvml" => "__builtin_ve_vl_vrand_vvml",
+    "llvm.ve.vl.vrcpd.vvl" => "__builtin_ve_vl_vrcpd_vvl",
+    "llvm.ve.vl.vrcpd.vvvl" => "__builtin_ve_vl_vrcpd_vvvl",
+    "llvm.ve.vl.vrcps.vvl" => "__builtin_ve_vl_vrcps_vvl",
+    "llvm.ve.vl.vrcps.vvvl" => "__builtin_ve_vl_vrcps_vvvl",
+    "llvm.ve.vl.vrmaxslfst.vvl" => "__builtin_ve_vl_vrmaxslfst_vvl",
+    "llvm.ve.vl.vrmaxslfst.vvvl" => "__builtin_ve_vl_vrmaxslfst_vvvl",
+    "llvm.ve.vl.vrmaxsllst.vvl" => "__builtin_ve_vl_vrmaxsllst_vvl",
+    "llvm.ve.vl.vrmaxsllst.vvvl" => "__builtin_ve_vl_vrmaxsllst_vvvl",
+    "llvm.ve.vl.vrmaxswfstsx.vvl" => "__builtin_ve_vl_vrmaxswfstsx_vvl",
+    "llvm.ve.vl.vrmaxswfstsx.vvvl" => "__builtin_ve_vl_vrmaxswfstsx_vvvl",
+    "llvm.ve.vl.vrmaxswfstzx.vvl" => "__builtin_ve_vl_vrmaxswfstzx_vvl",
+    "llvm.ve.vl.vrmaxswfstzx.vvvl" => "__builtin_ve_vl_vrmaxswfstzx_vvvl",
+    "llvm.ve.vl.vrmaxswlstsx.vvl" => "__builtin_ve_vl_vrmaxswlstsx_vvl",
+    "llvm.ve.vl.vrmaxswlstsx.vvvl" => "__builtin_ve_vl_vrmaxswlstsx_vvvl",
+    "llvm.ve.vl.vrmaxswlstzx.vvl" => "__builtin_ve_vl_vrmaxswlstzx_vvl",
+    "llvm.ve.vl.vrmaxswlstzx.vvvl" => "__builtin_ve_vl_vrmaxswlstzx_vvvl",
+    "llvm.ve.vl.vrminslfst.vvl" => "__builtin_ve_vl_vrminslfst_vvl",
+    "llvm.ve.vl.vrminslfst.vvvl" => "__builtin_ve_vl_vrminslfst_vvvl",
+    "llvm.ve.vl.vrminsllst.vvl" => "__builtin_ve_vl_vrminsllst_vvl",
+    "llvm.ve.vl.vrminsllst.vvvl" => "__builtin_ve_vl_vrminsllst_vvvl",
+    "llvm.ve.vl.vrminswfstsx.vvl" => "__builtin_ve_vl_vrminswfstsx_vvl",
+    "llvm.ve.vl.vrminswfstsx.vvvl" => "__builtin_ve_vl_vrminswfstsx_vvvl",
+    "llvm.ve.vl.vrminswfstzx.vvl" => "__builtin_ve_vl_vrminswfstzx_vvl",
+    "llvm.ve.vl.vrminswfstzx.vvvl" => "__builtin_ve_vl_vrminswfstzx_vvvl",
+    "llvm.ve.vl.vrminswlstsx.vvl" => "__builtin_ve_vl_vrminswlstsx_vvl",
+    "llvm.ve.vl.vrminswlstsx.vvvl" => "__builtin_ve_vl_vrminswlstsx_vvvl",
+    "llvm.ve.vl.vrminswlstzx.vvl" => "__builtin_ve_vl_vrminswlstzx_vvl",
+    "llvm.ve.vl.vrminswlstzx.vvvl" => "__builtin_ve_vl_vrminswlstzx_vvvl",
+    "llvm.ve.vl.vror.vvl" => "__builtin_ve_vl_vror_vvl",
+    "llvm.ve.vl.vror.vvml" => "__builtin_ve_vl_vror_vvml",
+    "llvm.ve.vl.vrsqrtd.vvl" => "__builtin_ve_vl_vrsqrtd_vvl",
+    "llvm.ve.vl.vrsqrtd.vvvl" => "__builtin_ve_vl_vrsqrtd_vvvl",
+    "llvm.ve.vl.vrsqrtdnex.vvl" => "__builtin_ve_vl_vrsqrtdnex_vvl",
+    "llvm.ve.vl.vrsqrtdnex.vvvl" => "__builtin_ve_vl_vrsqrtdnex_vvvl",
+    "llvm.ve.vl.vrsqrts.vvl" => "__builtin_ve_vl_vrsqrts_vvl",
+    "llvm.ve.vl.vrsqrts.vvvl" => "__builtin_ve_vl_vrsqrts_vvvl",
+    "llvm.ve.vl.vrsqrtsnex.vvl" => "__builtin_ve_vl_vrsqrtsnex_vvl",
+    "llvm.ve.vl.vrsqrtsnex.vvvl" => "__builtin_ve_vl_vrsqrtsnex_vvvl",
+    "llvm.ve.vl.vrxor.vvl" => "__builtin_ve_vl_vrxor_vvl",
+    "llvm.ve.vl.vrxor.vvml" => "__builtin_ve_vl_vrxor_vvml",
+    "llvm.ve.vl.vsc.vvssl" => "__builtin_ve_vl_vsc_vvssl",
+    "llvm.ve.vl.vsc.vvssml" => "__builtin_ve_vl_vsc_vvssml",
+    "llvm.ve.vl.vscl.vvssl" => "__builtin_ve_vl_vscl_vvssl",
+    "llvm.ve.vl.vscl.vvssml" => "__builtin_ve_vl_vscl_vvssml",
+    "llvm.ve.vl.vsclnc.vvssl" => "__builtin_ve_vl_vsclnc_vvssl",
+    "llvm.ve.vl.vsclnc.vvssml" => "__builtin_ve_vl_vsclnc_vvssml",
+    "llvm.ve.vl.vsclncot.vvssl" => "__builtin_ve_vl_vsclncot_vvssl",
+    "llvm.ve.vl.vsclncot.vvssml" => "__builtin_ve_vl_vsclncot_vvssml",
+    "llvm.ve.vl.vsclot.vvssl" => "__builtin_ve_vl_vsclot_vvssl",
+    "llvm.ve.vl.vsclot.vvssml" => "__builtin_ve_vl_vsclot_vvssml",
+    "llvm.ve.vl.vscnc.vvssl" => "__builtin_ve_vl_vscnc_vvssl",
+    "llvm.ve.vl.vscnc.vvssml" => "__builtin_ve_vl_vscnc_vvssml",
+    "llvm.ve.vl.vscncot.vvssl" => "__builtin_ve_vl_vscncot_vvssl",
+    "llvm.ve.vl.vscncot.vvssml" => "__builtin_ve_vl_vscncot_vvssml",
+    "llvm.ve.vl.vscot.vvssl" => "__builtin_ve_vl_vscot_vvssl",
+    "llvm.ve.vl.vscot.vvssml" => "__builtin_ve_vl_vscot_vvssml",
+    "llvm.ve.vl.vscu.vvssl" => "__builtin_ve_vl_vscu_vvssl",
+    "llvm.ve.vl.vscu.vvssml" => "__builtin_ve_vl_vscu_vvssml",
+    "llvm.ve.vl.vscunc.vvssl" => "__builtin_ve_vl_vscunc_vvssl",
+    "llvm.ve.vl.vscunc.vvssml" => "__builtin_ve_vl_vscunc_vvssml",
+    "llvm.ve.vl.vscuncot.vvssl" => "__builtin_ve_vl_vscuncot_vvssl",
+    "llvm.ve.vl.vscuncot.vvssml" => "__builtin_ve_vl_vscuncot_vvssml",
+    "llvm.ve.vl.vscuot.vvssl" => "__builtin_ve_vl_vscuot_vvssl",
+    "llvm.ve.vl.vscuot.vvssml" => "__builtin_ve_vl_vscuot_vvssml",
+    "llvm.ve.vl.vseq.vl" => "__builtin_ve_vl_vseq_vl",
+    "llvm.ve.vl.vseq.vvl" => "__builtin_ve_vl_vseq_vvl",
+    "llvm.ve.vl.vsfa.vvssl" => "__builtin_ve_vl_vsfa_vvssl",
+    "llvm.ve.vl.vsfa.vvssmvl" => "__builtin_ve_vl_vsfa_vvssmvl",
+    "llvm.ve.vl.vsfa.vvssvl" => "__builtin_ve_vl_vsfa_vvssvl",
+    "llvm.ve.vl.vshf.vvvsl" => "__builtin_ve_vl_vshf_vvvsl",
+    "llvm.ve.vl.vshf.vvvsvl" => "__builtin_ve_vl_vshf_vvvsvl",
+    "llvm.ve.vl.vslal.vvsl" => "__builtin_ve_vl_vslal_vvsl",
+    "llvm.ve.vl.vslal.vvsmvl" => "__builtin_ve_vl_vslal_vvsmvl",
+    "llvm.ve.vl.vslal.vvsvl" => "__builtin_ve_vl_vslal_vvsvl",
+    "llvm.ve.vl.vslal.vvvl" => "__builtin_ve_vl_vslal_vvvl",
+    "llvm.ve.vl.vslal.vvvmvl" => "__builtin_ve_vl_vslal_vvvmvl",
+    "llvm.ve.vl.vslal.vvvvl" => "__builtin_ve_vl_vslal_vvvvl",
+    "llvm.ve.vl.vslawsx.vvsl" => "__builtin_ve_vl_vslawsx_vvsl",
+    "llvm.ve.vl.vslawsx.vvsmvl" => "__builtin_ve_vl_vslawsx_vvsmvl",
+    "llvm.ve.vl.vslawsx.vvsvl" => "__builtin_ve_vl_vslawsx_vvsvl",
+    "llvm.ve.vl.vslawsx.vvvl" => "__builtin_ve_vl_vslawsx_vvvl",
+    "llvm.ve.vl.vslawsx.vvvmvl" => "__builtin_ve_vl_vslawsx_vvvmvl",
+    "llvm.ve.vl.vslawsx.vvvvl" => "__builtin_ve_vl_vslawsx_vvvvl",
+    "llvm.ve.vl.vslawzx.vvsl" => "__builtin_ve_vl_vslawzx_vvsl",
+    "llvm.ve.vl.vslawzx.vvsmvl" => "__builtin_ve_vl_vslawzx_vvsmvl",
+    "llvm.ve.vl.vslawzx.vvsvl" => "__builtin_ve_vl_vslawzx_vvsvl",
+    "llvm.ve.vl.vslawzx.vvvl" => "__builtin_ve_vl_vslawzx_vvvl",
+    "llvm.ve.vl.vslawzx.vvvmvl" => "__builtin_ve_vl_vslawzx_vvvmvl",
+    "llvm.ve.vl.vslawzx.vvvvl" => "__builtin_ve_vl_vslawzx_vvvvl",
+    "llvm.ve.vl.vsll.vvsl" => "__builtin_ve_vl_vsll_vvsl",
+    "llvm.ve.vl.vsll.vvsmvl" => "__builtin_ve_vl_vsll_vvsmvl",
+    "llvm.ve.vl.vsll.vvsvl" => "__builtin_ve_vl_vsll_vvsvl",
+    "llvm.ve.vl.vsll.vvvl" => "__builtin_ve_vl_vsll_vvvl",
+    "llvm.ve.vl.vsll.vvvmvl" => "__builtin_ve_vl_vsll_vvvmvl",
+    "llvm.ve.vl.vsll.vvvvl" => "__builtin_ve_vl_vsll_vvvvl",
+    "llvm.ve.vl.vsral.vvsl" => "__builtin_ve_vl_vsral_vvsl",
+    "llvm.ve.vl.vsral.vvsmvl" => "__builtin_ve_vl_vsral_vvsmvl",
+    "llvm.ve.vl.vsral.vvsvl" => "__builtin_ve_vl_vsral_vvsvl",
+    "llvm.ve.vl.vsral.vvvl" => "__builtin_ve_vl_vsral_vvvl",
+    "llvm.ve.vl.vsral.vvvmvl" => "__builtin_ve_vl_vsral_vvvmvl",
+    "llvm.ve.vl.vsral.vvvvl" => "__builtin_ve_vl_vsral_vvvvl",
+    "llvm.ve.vl.vsrawsx.vvsl" => "__builtin_ve_vl_vsrawsx_vvsl",
+    "llvm.ve.vl.vsrawsx.vvsmvl" => "__builtin_ve_vl_vsrawsx_vvsmvl",
+    "llvm.ve.vl.vsrawsx.vvsvl" => "__builtin_ve_vl_vsrawsx_vvsvl",
+    "llvm.ve.vl.vsrawsx.vvvl" => "__builtin_ve_vl_vsrawsx_vvvl",
+    "llvm.ve.vl.vsrawsx.vvvmvl" => "__builtin_ve_vl_vsrawsx_vvvmvl",
+    "llvm.ve.vl.vsrawsx.vvvvl" => "__builtin_ve_vl_vsrawsx_vvvvl",
+    "llvm.ve.vl.vsrawzx.vvsl" => "__builtin_ve_vl_vsrawzx_vvsl",
+    "llvm.ve.vl.vsrawzx.vvsmvl" => "__builtin_ve_vl_vsrawzx_vvsmvl",
+    "llvm.ve.vl.vsrawzx.vvsvl" => "__builtin_ve_vl_vsrawzx_vvsvl",
+    "llvm.ve.vl.vsrawzx.vvvl" => "__builtin_ve_vl_vsrawzx_vvvl",
+    "llvm.ve.vl.vsrawzx.vvvmvl" => "__builtin_ve_vl_vsrawzx_vvvmvl",
+    "llvm.ve.vl.vsrawzx.vvvvl" => "__builtin_ve_vl_vsrawzx_vvvvl",
+    "llvm.ve.vl.vsrl.vvsl" => "__builtin_ve_vl_vsrl_vvsl",
+    "llvm.ve.vl.vsrl.vvsmvl" => "__builtin_ve_vl_vsrl_vvsmvl",
+    "llvm.ve.vl.vsrl.vvsvl" => "__builtin_ve_vl_vsrl_vvsvl",
+    "llvm.ve.vl.vsrl.vvvl" => "__builtin_ve_vl_vsrl_vvvl",
+    "llvm.ve.vl.vsrl.vvvmvl" => "__builtin_ve_vl_vsrl_vvvmvl",
+    "llvm.ve.vl.vsrl.vvvvl" => "__builtin_ve_vl_vsrl_vvvvl",
+    "llvm.ve.vl.vst.vssl" => "__builtin_ve_vl_vst_vssl",
+    "llvm.ve.vl.vst.vssml" => "__builtin_ve_vl_vst_vssml",
+    "llvm.ve.vl.vst2d.vssl" => "__builtin_ve_vl_vst2d_vssl",
+    "llvm.ve.vl.vst2d.vssml" => "__builtin_ve_vl_vst2d_vssml",
+    "llvm.ve.vl.vst2dnc.vssl" => "__builtin_ve_vl_vst2dnc_vssl",
+    "llvm.ve.vl.vst2dnc.vssml" => "__builtin_ve_vl_vst2dnc_vssml",
+    "llvm.ve.vl.vst2dncot.vssl" => "__builtin_ve_vl_vst2dncot_vssl",
+    "llvm.ve.vl.vst2dncot.vssml" => "__builtin_ve_vl_vst2dncot_vssml",
+    "llvm.ve.vl.vst2dot.vssl" => "__builtin_ve_vl_vst2dot_vssl",
+    "llvm.ve.vl.vst2dot.vssml" => "__builtin_ve_vl_vst2dot_vssml",
+    "llvm.ve.vl.vstl.vssl" => "__builtin_ve_vl_vstl_vssl",
+    "llvm.ve.vl.vstl.vssml" => "__builtin_ve_vl_vstl_vssml",
+    "llvm.ve.vl.vstl2d.vssl" => "__builtin_ve_vl_vstl2d_vssl",
+    "llvm.ve.vl.vstl2d.vssml" => "__builtin_ve_vl_vstl2d_vssml",
+    "llvm.ve.vl.vstl2dnc.vssl" => "__builtin_ve_vl_vstl2dnc_vssl",
+    "llvm.ve.vl.vstl2dnc.vssml" => "__builtin_ve_vl_vstl2dnc_vssml",
+    "llvm.ve.vl.vstl2dncot.vssl" => "__builtin_ve_vl_vstl2dncot_vssl",
+    "llvm.ve.vl.vstl2dncot.vssml" => "__builtin_ve_vl_vstl2dncot_vssml",
+    "llvm.ve.vl.vstl2dot.vssl" => "__builtin_ve_vl_vstl2dot_vssl",
+    "llvm.ve.vl.vstl2dot.vssml" => "__builtin_ve_vl_vstl2dot_vssml",
+    "llvm.ve.vl.vstlnc.vssl" => "__builtin_ve_vl_vstlnc_vssl",
+    "llvm.ve.vl.vstlnc.vssml" => "__builtin_ve_vl_vstlnc_vssml",
+    "llvm.ve.vl.vstlncot.vssl" => "__builtin_ve_vl_vstlncot_vssl",
+    "llvm.ve.vl.vstlncot.vssml" => "__builtin_ve_vl_vstlncot_vssml",
+    "llvm.ve.vl.vstlot.vssl" => "__builtin_ve_vl_vstlot_vssl",
+    "llvm.ve.vl.vstlot.vssml" => "__builtin_ve_vl_vstlot_vssml",
+    "llvm.ve.vl.vstnc.vssl" => "__builtin_ve_vl_vstnc_vssl",
+    "llvm.ve.vl.vstnc.vssml" => "__builtin_ve_vl_vstnc_vssml",
+    "llvm.ve.vl.vstncot.vssl" => "__builtin_ve_vl_vstncot_vssl",
+    "llvm.ve.vl.vstncot.vssml" => "__builtin_ve_vl_vstncot_vssml",
+    "llvm.ve.vl.vstot.vssl" => "__builtin_ve_vl_vstot_vssl",
+    "llvm.ve.vl.vstot.vssml" => "__builtin_ve_vl_vstot_vssml",
+    "llvm.ve.vl.vstu.vssl" => "__builtin_ve_vl_vstu_vssl",
+    "llvm.ve.vl.vstu.vssml" => "__builtin_ve_vl_vstu_vssml",
+    "llvm.ve.vl.vstu2d.vssl" => "__builtin_ve_vl_vstu2d_vssl",
+    "llvm.ve.vl.vstu2d.vssml" => "__builtin_ve_vl_vstu2d_vssml",
+    "llvm.ve.vl.vstu2dnc.vssl" => "__builtin_ve_vl_vstu2dnc_vssl",
+    "llvm.ve.vl.vstu2dnc.vssml" => "__builtin_ve_vl_vstu2dnc_vssml",
+    "llvm.ve.vl.vstu2dncot.vssl" => "__builtin_ve_vl_vstu2dncot_vssl",
+    "llvm.ve.vl.vstu2dncot.vssml" => "__builtin_ve_vl_vstu2dncot_vssml",
+    "llvm.ve.vl.vstu2dot.vssl" => "__builtin_ve_vl_vstu2dot_vssl",
+    "llvm.ve.vl.vstu2dot.vssml" => "__builtin_ve_vl_vstu2dot_vssml",
+    "llvm.ve.vl.vstunc.vssl" => "__builtin_ve_vl_vstunc_vssl",
+    "llvm.ve.vl.vstunc.vssml" => "__builtin_ve_vl_vstunc_vssml",
+    "llvm.ve.vl.vstuncot.vssl" => "__builtin_ve_vl_vstuncot_vssl",
+    "llvm.ve.vl.vstuncot.vssml" => "__builtin_ve_vl_vstuncot_vssml",
+    "llvm.ve.vl.vstuot.vssl" => "__builtin_ve_vl_vstuot_vssl",
+    "llvm.ve.vl.vstuot.vssml" => "__builtin_ve_vl_vstuot_vssml",
+    "llvm.ve.vl.vsubsl.vsvl" => "__builtin_ve_vl_vsubsl_vsvl",
+    "llvm.ve.vl.vsubsl.vsvmvl" => "__builtin_ve_vl_vsubsl_vsvmvl",
+    "llvm.ve.vl.vsubsl.vsvvl" => "__builtin_ve_vl_vsubsl_vsvvl",
+    "llvm.ve.vl.vsubsl.vvvl" => "__builtin_ve_vl_vsubsl_vvvl",
+    "llvm.ve.vl.vsubsl.vvvmvl" => "__builtin_ve_vl_vsubsl_vvvmvl",
+    "llvm.ve.vl.vsubsl.vvvvl" => "__builtin_ve_vl_vsubsl_vvvvl",
+    "llvm.ve.vl.vsubswsx.vsvl" => "__builtin_ve_vl_vsubswsx_vsvl",
+    "llvm.ve.vl.vsubswsx.vsvmvl" => "__builtin_ve_vl_vsubswsx_vsvmvl",
+    "llvm.ve.vl.vsubswsx.vsvvl" => "__builtin_ve_vl_vsubswsx_vsvvl",
+    "llvm.ve.vl.vsubswsx.vvvl" => "__builtin_ve_vl_vsubswsx_vvvl",
+    "llvm.ve.vl.vsubswsx.vvvmvl" => "__builtin_ve_vl_vsubswsx_vvvmvl",
+    "llvm.ve.vl.vsubswsx.vvvvl" => "__builtin_ve_vl_vsubswsx_vvvvl",
+    "llvm.ve.vl.vsubswzx.vsvl" => "__builtin_ve_vl_vsubswzx_vsvl",
+    "llvm.ve.vl.vsubswzx.vsvmvl" => "__builtin_ve_vl_vsubswzx_vsvmvl",
+    "llvm.ve.vl.vsubswzx.vsvvl" => "__builtin_ve_vl_vsubswzx_vsvvl",
+    "llvm.ve.vl.vsubswzx.vvvl" => "__builtin_ve_vl_vsubswzx_vvvl",
+    "llvm.ve.vl.vsubswzx.vvvmvl" => "__builtin_ve_vl_vsubswzx_vvvmvl",
+    "llvm.ve.vl.vsubswzx.vvvvl" => "__builtin_ve_vl_vsubswzx_vvvvl",
+    "llvm.ve.vl.vsubul.vsvl" => "__builtin_ve_vl_vsubul_vsvl",
+    "llvm.ve.vl.vsubul.vsvmvl" => "__builtin_ve_vl_vsubul_vsvmvl",
+    "llvm.ve.vl.vsubul.vsvvl" => "__builtin_ve_vl_vsubul_vsvvl",
+    "llvm.ve.vl.vsubul.vvvl" => "__builtin_ve_vl_vsubul_vvvl",
+    "llvm.ve.vl.vsubul.vvvmvl" => "__builtin_ve_vl_vsubul_vvvmvl",
+    "llvm.ve.vl.vsubul.vvvvl" => "__builtin_ve_vl_vsubul_vvvvl",
+    "llvm.ve.vl.vsubuw.vsvl" => "__builtin_ve_vl_vsubuw_vsvl",
+    "llvm.ve.vl.vsubuw.vsvmvl" => "__builtin_ve_vl_vsubuw_vsvmvl",
+    "llvm.ve.vl.vsubuw.vsvvl" => "__builtin_ve_vl_vsubuw_vsvvl",
+    "llvm.ve.vl.vsubuw.vvvl" => "__builtin_ve_vl_vsubuw_vvvl",
+    "llvm.ve.vl.vsubuw.vvvmvl" => "__builtin_ve_vl_vsubuw_vvvmvl",
+    "llvm.ve.vl.vsubuw.vvvvl" => "__builtin_ve_vl_vsubuw_vvvvl",
+    "llvm.ve.vl.vsuml.vvl" => "__builtin_ve_vl_vsuml_vvl",
+    "llvm.ve.vl.vsuml.vvml" => "__builtin_ve_vl_vsuml_vvml",
+    "llvm.ve.vl.vsumwsx.vvl" => "__builtin_ve_vl_vsumwsx_vvl",
+    "llvm.ve.vl.vsumwsx.vvml" => "__builtin_ve_vl_vsumwsx_vvml",
+    "llvm.ve.vl.vsumwzx.vvl" => "__builtin_ve_vl_vsumwzx_vvl",
+    "llvm.ve.vl.vsumwzx.vvml" => "__builtin_ve_vl_vsumwzx_vvml",
+    "llvm.ve.vl.vxor.vsvl" => "__builtin_ve_vl_vxor_vsvl",
+    "llvm.ve.vl.vxor.vsvmvl" => "__builtin_ve_vl_vxor_vsvmvl",
+    "llvm.ve.vl.vxor.vsvvl" => "__builtin_ve_vl_vxor_vsvvl",
+    "llvm.ve.vl.vxor.vvvl" => "__builtin_ve_vl_vxor_vvvl",
+    "llvm.ve.vl.vxor.vvvmvl" => "__builtin_ve_vl_vxor_vvvmvl",
+    "llvm.ve.vl.vxor.vvvvl" => "__builtin_ve_vl_vxor_vvvvl",
+    "llvm.ve.vl.xorm.MMM" => "__builtin_ve_vl_xorm_MMM",
+    "llvm.ve.vl.xorm.mmm" => "__builtin_ve_vl_xorm_mmm",
     // x86
     "llvm.x86.3dnow.pavgusb" => "__builtin_ia32_pavgusb",
     "llvm.x86.3dnow.pf2id" => "__builtin_ia32_pf2id",
@@ -3430,6 +5706,10 @@ match name {
     "llvm.x86.3dnowa.pfnacc" => "__builtin_ia32_pfnacc",
     "llvm.x86.3dnowa.pfpnacc" => "__builtin_ia32_pfpnacc",
     "llvm.x86.3dnowa.pi2fw" => "__builtin_ia32_pi2fw",
+    "llvm.x86.aadd32" => "__builtin_ia32_aadd32",
+    "llvm.x86.aadd64" => "__builtin_ia32_aadd64",
+    "llvm.x86.aand32" => "__builtin_ia32_aand32",
+    "llvm.x86.aand64" => "__builtin_ia32_aand64",
     "llvm.x86.addcarry.u32" => "__builtin_ia32_addcarry_u32",
     "llvm.x86.addcarry.u64" => "__builtin_ia32_addcarry_u64",
     "llvm.x86.addcarryx.u32" => "__builtin_ia32_addcarryx_u32",
@@ -3448,6 +5728,8 @@ match name {
     "llvm.x86.aesni.aesenclast.512" => "__builtin_ia32_aesenclast512",
     "llvm.x86.aesni.aesimc" => "__builtin_ia32_aesimc128",
     "llvm.x86.aesni.aeskeygenassist" => "__builtin_ia32_aeskeygenassist128",
+    "llvm.x86.aor32" => "__builtin_ia32_aor32",
+    "llvm.x86.aor64" => "__builtin_ia32_aor64",
     "llvm.x86.avx.addsub.pd.256" => "__builtin_ia32_addsubpd256",
     "llvm.x86.avx.addsub.ps.256" => "__builtin_ia32_addsubps256",
     "llvm.x86.avx.blend.pd.256" => "__builtin_ia32_blendpd256",
@@ -3660,6 +5942,18 @@ match name {
     "llvm.x86.avx2.vbroadcast.ss.ps.256" => "__builtin_ia32_vbroadcastss_ps256",
     "llvm.x86.avx2.vextracti128" => "__builtin_ia32_extract128i256",
     "llvm.x86.avx2.vinserti128" => "__builtin_ia32_insert128i256",
+    "llvm.x86.avx2.vpdpbssd.128" => "__builtin_ia32_vpdpbssd128",
+    "llvm.x86.avx2.vpdpbssd.256" => "__builtin_ia32_vpdpbssd256",
+    "llvm.x86.avx2.vpdpbssds.128" => "__builtin_ia32_vpdpbssds128",
+    "llvm.x86.avx2.vpdpbssds.256" => "__builtin_ia32_vpdpbssds256",
+    "llvm.x86.avx2.vpdpbsud.128" => "__builtin_ia32_vpdpbsud128",
+    "llvm.x86.avx2.vpdpbsud.256" => "__builtin_ia32_vpdpbsud256",
+    "llvm.x86.avx2.vpdpbsuds.128" => "__builtin_ia32_vpdpbsuds128",
+    "llvm.x86.avx2.vpdpbsuds.256" => "__builtin_ia32_vpdpbsuds256",
+    "llvm.x86.avx2.vpdpbuud.128" => "__builtin_ia32_vpdpbuud128",
+    "llvm.x86.avx2.vpdpbuud.256" => "__builtin_ia32_vpdpbuud256",
+    "llvm.x86.avx2.vpdpbuuds.128" => "__builtin_ia32_vpdpbuuds128",
+    "llvm.x86.avx2.vpdpbuuds.256" => "__builtin_ia32_vpdpbuuds256",
     "llvm.x86.avx2.vperm2i128" => "__builtin_ia32_permti256",
     "llvm.x86.avx512.add.pd.512" => "__builtin_ia32_addpd512",
     "llvm.x86.avx512.add.ps.512" => "__builtin_ia32_addps512",
@@ -3779,8 +6073,8 @@ match name {
     "llvm.x86.avx512.mask.add.ps.128" => "__builtin_ia32_addps128_mask",
     "llvm.x86.avx512.mask.add.ps.256" => "__builtin_ia32_addps256_mask",
     "llvm.x86.avx512.mask.add.ps.512" => "__builtin_ia32_addps512_mask",
-    "llvm.x86.avx512.mask.add.sd.round" => "__builtin_ia32_addsd_round_mask",
-    "llvm.x86.avx512.mask.add.ss.round" => "__builtin_ia32_addss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.add.sd.round" => "__builtin_ia32_addsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.add.ss.round" => "__builtin_ia32_addss_round_mask",
     "llvm.x86.avx512.mask.and.pd.128" => "__builtin_ia32_andpd128_mask",
     "llvm.x86.avx512.mask.and.pd.256" => "__builtin_ia32_andpd256_mask",
     "llvm.x86.avx512.mask.and.pd.512" => "__builtin_ia32_andpd512_mask",
@@ -3894,8 +6188,8 @@ match name {
     "llvm.x86.avx512.mask.cvtqq2ps.128" => "__builtin_ia32_cvtqq2ps128_mask",
     "llvm.x86.avx512.mask.cvtqq2ps.256" => "__builtin_ia32_cvtqq2ps256_mask",
     "llvm.x86.avx512.mask.cvtqq2ps.512" => "__builtin_ia32_cvtqq2ps512_mask",
-    "llvm.x86.avx512.mask.cvtsd2ss.round" => "__builtin_ia32_cvtsd2ss_round_mask",
-    "llvm.x86.avx512.mask.cvtss2sd.round" => "__builtin_ia32_cvtss2sd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.cvtsd2ss.round" => "__builtin_ia32_cvtsd2ss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.cvtss2sd.round" => "__builtin_ia32_cvtss2sd_round_mask",
     "llvm.x86.avx512.mask.cvttpd2dq.128" => "__builtin_ia32_cvttpd2dq128_mask",
     "llvm.x86.avx512.mask.cvttpd2dq.256" => "__builtin_ia32_cvttpd2dq256_mask",
     "llvm.x86.avx512.mask.cvttpd2dq.512" => "__builtin_ia32_cvttpd2dq512_mask",
@@ -3941,8 +6235,8 @@ match name {
     "llvm.x86.avx512.mask.div.ps.128" => "__builtin_ia32_divps_mask",
     "llvm.x86.avx512.mask.div.ps.256" => "__builtin_ia32_divps256_mask",
     "llvm.x86.avx512.mask.div.ps.512" => "__builtin_ia32_divps512_mask",
-    "llvm.x86.avx512.mask.div.sd.round" => "__builtin_ia32_divsd_round_mask",
-    "llvm.x86.avx512.mask.div.ss.round" => "__builtin_ia32_divss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.div.sd.round" => "__builtin_ia32_divsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.div.ss.round" => "__builtin_ia32_divss_round_mask",
     "llvm.x86.avx512.mask.expand.d.128" => "__builtin_ia32_expandsi128_mask",
     "llvm.x86.avx512.mask.expand.d.256" => "__builtin_ia32_expandsi256_mask",
     "llvm.x86.avx512.mask.expand.d.512" => "__builtin_ia32_expandsi512_mask",
@@ -3989,16 +6283,16 @@ match name {
     "llvm.x86.avx512.mask.getexp.ps.128" => "__builtin_ia32_getexpps128_mask",
     "llvm.x86.avx512.mask.getexp.ps.256" => "__builtin_ia32_getexpps256_mask",
     "llvm.x86.avx512.mask.getexp.ps.512" => "__builtin_ia32_getexpps512_mask",
-    "llvm.x86.avx512.mask.getexp.sd" => "__builtin_ia32_getexpsd128_round_mask",
-    "llvm.x86.avx512.mask.getexp.ss" => "__builtin_ia32_getexpss128_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.getexp.sd" => "__builtin_ia32_getexpsd128_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.getexp.ss" => "__builtin_ia32_getexpss128_round_mask",
     "llvm.x86.avx512.mask.getmant.pd.128" => "__builtin_ia32_getmantpd128_mask",
     "llvm.x86.avx512.mask.getmant.pd.256" => "__builtin_ia32_getmantpd256_mask",
     "llvm.x86.avx512.mask.getmant.pd.512" => "__builtin_ia32_getmantpd512_mask",
     "llvm.x86.avx512.mask.getmant.ps.128" => "__builtin_ia32_getmantps128_mask",
     "llvm.x86.avx512.mask.getmant.ps.256" => "__builtin_ia32_getmantps256_mask",
     "llvm.x86.avx512.mask.getmant.ps.512" => "__builtin_ia32_getmantps512_mask",
-    "llvm.x86.avx512.mask.getmant.sd" => "__builtin_ia32_getmantsd_round_mask",
-    "llvm.x86.avx512.mask.getmant.ss" => "__builtin_ia32_getmantss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.getmant.sd" => "__builtin_ia32_getmantsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.getmant.ss" => "__builtin_ia32_getmantss_round_mask",
     "llvm.x86.avx512.mask.insertf32x4.256" => "__builtin_ia32_insertf32x4_256_mask",
     "llvm.x86.avx512.mask.insertf32x4.512" => "__builtin_ia32_insertf32x4_mask",
     "llvm.x86.avx512.mask.insertf32x8.512" => "__builtin_ia32_insertf32x8_mask",
@@ -4023,16 +6317,16 @@ match name {
     "llvm.x86.avx512.mask.max.ps.128" => "__builtin_ia32_maxps_mask",
     "llvm.x86.avx512.mask.max.ps.256" => "__builtin_ia32_maxps256_mask",
     "llvm.x86.avx512.mask.max.ps.512" => "__builtin_ia32_maxps512_mask",
-    "llvm.x86.avx512.mask.max.sd.round" => "__builtin_ia32_maxsd_round_mask",
-    "llvm.x86.avx512.mask.max.ss.round" => "__builtin_ia32_maxss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.max.sd.round" => "__builtin_ia32_maxsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.max.ss.round" => "__builtin_ia32_maxss_round_mask",
     "llvm.x86.avx512.mask.min.pd.128" => "__builtin_ia32_minpd_mask",
     "llvm.x86.avx512.mask.min.pd.256" => "__builtin_ia32_minpd256_mask",
     "llvm.x86.avx512.mask.min.pd.512" => "__builtin_ia32_minpd512_mask",
     "llvm.x86.avx512.mask.min.ps.128" => "__builtin_ia32_minps_mask",
     "llvm.x86.avx512.mask.min.ps.256" => "__builtin_ia32_minps256_mask",
     "llvm.x86.avx512.mask.min.ps.512" => "__builtin_ia32_minps512_mask",
-    "llvm.x86.avx512.mask.min.sd.round" => "__builtin_ia32_minsd_round_mask",
-    "llvm.x86.avx512.mask.min.ss.round" => "__builtin_ia32_minss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.min.sd.round" => "__builtin_ia32_minsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.min.ss.round" => "__builtin_ia32_minss_round_mask",
     "llvm.x86.avx512.mask.move.sd" => "__builtin_ia32_movsd_mask",
     "llvm.x86.avx512.mask.move.ss" => "__builtin_ia32_movss_mask",
     "llvm.x86.avx512.mask.mul.pd.128" => "__builtin_ia32_mulpd_mask",
@@ -4041,8 +6335,8 @@ match name {
     "llvm.x86.avx512.mask.mul.ps.128" => "__builtin_ia32_mulps_mask",
     "llvm.x86.avx512.mask.mul.ps.256" => "__builtin_ia32_mulps256_mask",
     "llvm.x86.avx512.mask.mul.ps.512" => "__builtin_ia32_mulps512_mask",
-    "llvm.x86.avx512.mask.mul.sd.round" => "__builtin_ia32_mulsd_round_mask",
-    "llvm.x86.avx512.mask.mul.ss.round" => "__builtin_ia32_mulss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.mul.sd.round" => "__builtin_ia32_mulsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.mul.ss.round" => "__builtin_ia32_mulss_round_mask",
     "llvm.x86.avx512.mask.or.pd.128" => "__builtin_ia32_orpd128_mask",
     "llvm.x86.avx512.mask.or.pd.256" => "__builtin_ia32_orpd256_mask",
     "llvm.x86.avx512.mask.or.pd.512" => "__builtin_ia32_orpd512_mask",
@@ -4527,8 +6821,8 @@ match name {
     "llvm.x86.avx512.mask.range.ps.128" => "__builtin_ia32_rangeps128_mask",
     "llvm.x86.avx512.mask.range.ps.256" => "__builtin_ia32_rangeps256_mask",
     "llvm.x86.avx512.mask.range.ps.512" => "__builtin_ia32_rangeps512_mask",
-    "llvm.x86.avx512.mask.range.sd" => "__builtin_ia32_rangesd128_round_mask",
-    "llvm.x86.avx512.mask.range.ss" => "__builtin_ia32_rangess128_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.range.sd" => "__builtin_ia32_rangesd128_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.range.ss" => "__builtin_ia32_rangess128_round_mask",
     "llvm.x86.avx512.mask.reduce.pd.128" => "__builtin_ia32_reducepd128_mask",
     "llvm.x86.avx512.mask.reduce.pd.256" => "__builtin_ia32_reducepd256_mask",
     "llvm.x86.avx512.mask.reduce.pd.512" => "__builtin_ia32_reducepd512_mask",
@@ -4543,16 +6837,16 @@ match name {
     "llvm.x86.avx512.mask.rndscale.ps.128" => "__builtin_ia32_rndscaleps_128_mask",
     "llvm.x86.avx512.mask.rndscale.ps.256" => "__builtin_ia32_rndscaleps_256_mask",
     "llvm.x86.avx512.mask.rndscale.ps.512" => "__builtin_ia32_rndscaleps_mask",
-    "llvm.x86.avx512.mask.rndscale.sd" => "__builtin_ia32_rndscalesd_round_mask",
-    "llvm.x86.avx512.mask.rndscale.ss" => "__builtin_ia32_rndscaless_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.rndscale.sd" => "__builtin_ia32_rndscalesd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.rndscale.ss" => "__builtin_ia32_rndscaless_round_mask",
     "llvm.x86.avx512.mask.scalef.pd.128" => "__builtin_ia32_scalefpd128_mask",
     "llvm.x86.avx512.mask.scalef.pd.256" => "__builtin_ia32_scalefpd256_mask",
     "llvm.x86.avx512.mask.scalef.pd.512" => "__builtin_ia32_scalefpd512_mask",
     "llvm.x86.avx512.mask.scalef.ps.128" => "__builtin_ia32_scalefps128_mask",
     "llvm.x86.avx512.mask.scalef.ps.256" => "__builtin_ia32_scalefps256_mask",
     "llvm.x86.avx512.mask.scalef.ps.512" => "__builtin_ia32_scalefps512_mask",
-    "llvm.x86.avx512.mask.scalef.sd" => "__builtin_ia32_scalefsd_round_mask",
-    "llvm.x86.avx512.mask.scalef.ss" => "__builtin_ia32_scalefss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.scalef.sd" => "__builtin_ia32_scalefsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.scalef.ss" => "__builtin_ia32_scalefss_round_mask",
     "llvm.x86.avx512.mask.shuf.f32x4" => "__builtin_ia32_shuf_f32x4_mask",
     "llvm.x86.avx512.mask.shuf.f32x4.256" => "__builtin_ia32_shuf_f32x4_256_mask",
     "llvm.x86.avx512.mask.shuf.f64x2" => "__builtin_ia32_shuf_f64x2_mask",
@@ -4573,8 +6867,8 @@ match name {
     "llvm.x86.avx512.mask.sqrt.ps.128" => "__builtin_ia32_sqrtps128_mask",
     "llvm.x86.avx512.mask.sqrt.ps.256" => "__builtin_ia32_sqrtps256_mask",
     "llvm.x86.avx512.mask.sqrt.ps.512" => "__builtin_ia32_sqrtps512_mask",
-    "llvm.x86.avx512.mask.sqrt.sd" => "__builtin_ia32_sqrtsd_round_mask",
-    "llvm.x86.avx512.mask.sqrt.ss" => "__builtin_ia32_sqrtss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.sqrt.sd" => "__builtin_ia32_sqrtsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.sqrt.ss" => "__builtin_ia32_sqrtss_round_mask",
     "llvm.x86.avx512.mask.store.ss" => "__builtin_ia32_storess_mask",
     "llvm.x86.avx512.mask.storeu.d.512" => "__builtin_ia32_storedqusi512_mask",
     "llvm.x86.avx512.mask.storeu.pd.512" => "__builtin_ia32_storeupd512_mask",
@@ -4586,8 +6880,8 @@ match name {
     "llvm.x86.avx512.mask.sub.ps.128" => "__builtin_ia32_subps128_mask",
     "llvm.x86.avx512.mask.sub.ps.256" => "__builtin_ia32_subps256_mask",
     "llvm.x86.avx512.mask.sub.ps.512" => "__builtin_ia32_subps512_mask",
-    "llvm.x86.avx512.mask.sub.sd.round" => "__builtin_ia32_subsd_round_mask",
-    "llvm.x86.avx512.mask.sub.ss.round" => "__builtin_ia32_subss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.sub.sd.round" => "__builtin_ia32_subsd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.mask.sub.ss.round" => "__builtin_ia32_subss_round_mask",
     "llvm.x86.avx512.mask.valign.d.128" => "__builtin_ia32_alignd128_mask",
     "llvm.x86.avx512.mask.valign.d.256" => "__builtin_ia32_alignd256_mask",
     "llvm.x86.avx512.mask.valign.d.512" => "__builtin_ia32_alignd512_mask",
@@ -4905,9 +7199,9 @@ match name {
     "llvm.x86.avx512.rcp14.ss" => "__builtin_ia32_rcp14ss_mask",
     "llvm.x86.avx512.rcp28.pd" => "__builtin_ia32_rcp28pd_mask",
     "llvm.x86.avx512.rcp28.ps" => "__builtin_ia32_rcp28ps_mask",
-    "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_round_mask",
     // [DUPLICATE]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask",
-    "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_round_mask",
     // [DUPLICATE]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask",
     "llvm.x86.avx512.rndscale.sd" => "__builtin_ia32_rndscalesd",
     "llvm.x86.avx512.rndscale.ss" => "__builtin_ia32_rndscaless",
@@ -4921,9 +7215,9 @@ match name {
     "llvm.x86.avx512.rsqrt14.ss" => "__builtin_ia32_rsqrt14ss_mask",
     "llvm.x86.avx512.rsqrt28.pd" => "__builtin_ia32_rsqrt28pd_mask",
     "llvm.x86.avx512.rsqrt28.ps" => "__builtin_ia32_rsqrt28ps_mask",
-    "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_round_mask",
     // [DUPLICATE]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask",
-    "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_round_mask",
     // [DUPLICATE]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask",
     "llvm.x86.avx512.scatter.dpd.512" => "__builtin_ia32_scattersiv8df",
     "llvm.x86.avx512.scatter.dpi.512" => "__builtin_ia32_scattersiv16si",
@@ -5021,21 +7315,21 @@ match name {
     "llvm.x86.avx512bf16.dpbf16ps.512" => "__builtin_ia32_dpbf16ps_512",
     "llvm.x86.avx512fp16.add.ph.512" => "__builtin_ia32_addph512",
     "llvm.x86.avx512fp16.div.ph.512" => "__builtin_ia32_divph512",
-    "llvm.x86.avx512fp16.mask.add.sh.round" => "__builtin_ia32_addsh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.add.sh.round" => "__builtin_ia32_addsh_round_mask",
     "llvm.x86.avx512fp16.mask.cmp.sh" => "__builtin_ia32_cmpsh_mask",
-    "llvm.x86.avx512fp16.mask.div.sh.round" => "__builtin_ia32_divsh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.div.sh.round" => "__builtin_ia32_divsh_round_mask",
     "llvm.x86.avx512fp16.mask.fpclass.sh" => "__builtin_ia32_fpclasssh_mask",
     "llvm.x86.avx512fp16.mask.getexp.ph.128" => "__builtin_ia32_getexpph128_mask",
     "llvm.x86.avx512fp16.mask.getexp.ph.256" => "__builtin_ia32_getexpph256_mask",
     "llvm.x86.avx512fp16.mask.getexp.ph.512" => "__builtin_ia32_getexpph512_mask",
-    "llvm.x86.avx512fp16.mask.getexp.sh" => "__builtin_ia32_getexpsh128_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.getexp.sh" => "__builtin_ia32_getexpsh128_round_mask",
     "llvm.x86.avx512fp16.mask.getmant.ph.128" => "__builtin_ia32_getmantph128_mask",
     "llvm.x86.avx512fp16.mask.getmant.ph.256" => "__builtin_ia32_getmantph256_mask",
     "llvm.x86.avx512fp16.mask.getmant.ph.512" => "__builtin_ia32_getmantph512_mask",
-    "llvm.x86.avx512fp16.mask.getmant.sh" => "__builtin_ia32_getmantsh_round_mask",
-    "llvm.x86.avx512fp16.mask.max.sh.round" => "__builtin_ia32_maxsh_round_mask",
-    "llvm.x86.avx512fp16.mask.min.sh.round" => "__builtin_ia32_minsh_round_mask",
-    "llvm.x86.avx512fp16.mask.mul.sh.round" => "__builtin_ia32_mulsh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.getmant.sh" => "__builtin_ia32_getmantsh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.max.sh.round" => "__builtin_ia32_maxsh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.min.sh.round" => "__builtin_ia32_minsh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.mul.sh.round" => "__builtin_ia32_mulsh_round_mask",
     "llvm.x86.avx512fp16.mask.rcp.ph.128" => "__builtin_ia32_rcpph128_mask",
     "llvm.x86.avx512fp16.mask.rcp.ph.256" => "__builtin_ia32_rcpph256_mask",
     "llvm.x86.avx512fp16.mask.rcp.ph.512" => "__builtin_ia32_rcpph512_mask",
@@ -5047,7 +7341,7 @@ match name {
     "llvm.x86.avx512fp16.mask.rndscale.ph.128" => "__builtin_ia32_rndscaleph_128_mask",
     "llvm.x86.avx512fp16.mask.rndscale.ph.256" => "__builtin_ia32_rndscaleph_256_mask",
     "llvm.x86.avx512fp16.mask.rndscale.ph.512" => "__builtin_ia32_rndscaleph_mask",
-    "llvm.x86.avx512fp16.mask.rndscale.sh" => "__builtin_ia32_rndscalesh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.rndscale.sh" => "__builtin_ia32_rndscalesh_round_mask",
     "llvm.x86.avx512fp16.mask.rsqrt.ph.128" => "__builtin_ia32_rsqrtph128_mask",
     "llvm.x86.avx512fp16.mask.rsqrt.ph.256" => "__builtin_ia32_rsqrtph256_mask",
     "llvm.x86.avx512fp16.mask.rsqrt.ph.512" => "__builtin_ia32_rsqrtph512_mask",
@@ -5055,8 +7349,8 @@ match name {
     "llvm.x86.avx512fp16.mask.scalef.ph.128" => "__builtin_ia32_scalefph128_mask",
     "llvm.x86.avx512fp16.mask.scalef.ph.256" => "__builtin_ia32_scalefph256_mask",
     "llvm.x86.avx512fp16.mask.scalef.ph.512" => "__builtin_ia32_scalefph512_mask",
-    "llvm.x86.avx512fp16.mask.scalef.sh" => "__builtin_ia32_scalefsh_round_mask",
-    "llvm.x86.avx512fp16.mask.sub.sh.round" => "__builtin_ia32_subsh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.scalef.sh" => "__builtin_ia32_scalefsh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.sub.sh.round" => "__builtin_ia32_subsh_round_mask",
     "llvm.x86.avx512fp16.mask.vcvtdq2ph.128" => "__builtin_ia32_vcvtdq2ph128_mask",
     "llvm.x86.avx512fp16.mask.vcvtpd2ph.128" => "__builtin_ia32_vcvtpd2ph128_mask",
     "llvm.x86.avx512fp16.mask.vcvtpd2ph.256" => "__builtin_ia32_vcvtpd2ph256_mask",
@@ -5090,10 +7384,10 @@ match name {
     "llvm.x86.avx512fp16.mask.vcvtps2phx.512" => "__builtin_ia32_vcvtps2phx512_mask",
     "llvm.x86.avx512fp16.mask.vcvtqq2ph.128" => "__builtin_ia32_vcvtqq2ph128_mask",
     "llvm.x86.avx512fp16.mask.vcvtqq2ph.256" => "__builtin_ia32_vcvtqq2ph256_mask",
-    "llvm.x86.avx512fp16.mask.vcvtsd2sh.round" => "__builtin_ia32_vcvtsd2sh_round_mask",
-    "llvm.x86.avx512fp16.mask.vcvtsh2sd.round" => "__builtin_ia32_vcvtsh2sd_round_mask",
-    "llvm.x86.avx512fp16.mask.vcvtsh2ss.round" => "__builtin_ia32_vcvtsh2ss_round_mask",
-    "llvm.x86.avx512fp16.mask.vcvtss2sh.round" => "__builtin_ia32_vcvtss2sh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.vcvtsd2sh.round" => "__builtin_ia32_vcvtsd2sh_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.vcvtsh2sd.round" => "__builtin_ia32_vcvtsh2sd_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.vcvtsh2ss.round" => "__builtin_ia32_vcvtsh2ss_round_mask",
+    // [INVALID CONVERSION]: "llvm.x86.avx512fp16.mask.vcvtss2sh.round" => "__builtin_ia32_vcvtss2sh_round_mask",
     "llvm.x86.avx512fp16.mask.vcvttph2dq.128" => "__builtin_ia32_vcvttph2dq128_mask",
     "llvm.x86.avx512fp16.mask.vcvttph2dq.256" => "__builtin_ia32_vcvttph2dq256_mask",
     "llvm.x86.avx512fp16.mask.vcvttph2dq.512" => "__builtin_ia32_vcvttph2dq512_mask",
@@ -5162,6 +7456,8 @@ match name {
     "llvm.x86.avx512fp16.vcvtusi642sh" => "__builtin_ia32_vcvtusi642sh",
     "llvm.x86.avx512fp16.vfmaddsub.ph.128" => "__builtin_ia32_vfmaddsubph",
     "llvm.x86.avx512fp16.vfmaddsub.ph.256" => "__builtin_ia32_vfmaddsubph256",
+    "llvm.x86.axor32" => "__builtin_ia32_axor32",
+    "llvm.x86.axor64" => "__builtin_ia32_axor64",
     "llvm.x86.bmi.bextr.32" => "__builtin_ia32_bextr_u32",
     "llvm.x86.bmi.bextr.64" => "__builtin_ia32_bextr_u64",
     "llvm.x86.bmi.bzhi.32" => "__builtin_ia32_bzhi_si",
@@ -5176,6 +7472,8 @@ match name {
     "llvm.x86.clui" => "__builtin_ia32_clui",
     "llvm.x86.clwb" => "__builtin_ia32_clwb",
     "llvm.x86.clzero" => "__builtin_ia32_clzero",
+    "llvm.x86.cmpccxadd32" => "__builtin_ia32_cmpccxadd32",
+    "llvm.x86.cmpccxadd64" => "__builtin_ia32_cmpccxadd64",
     "llvm.x86.directstore32" => "__builtin_ia32_directstore_u32",
     "llvm.x86.directstore64" => "__builtin_ia32_directstore_u64",
     "llvm.x86.enqcmd" => "__builtin_ia32_enqcmd",
@@ -5329,6 +7627,7 @@ match name {
     "llvm.x86.rdpid" => "__builtin_ia32_rdpid",
     "llvm.x86.rdpkru" => "__builtin_ia32_rdpkru",
     "llvm.x86.rdpmc" => "__builtin_ia32_rdpmc",
+    "llvm.x86.rdpru" => "__builtin_ia32_rdpru",
     "llvm.x86.rdsspd" => "__builtin_ia32_rdsspd",
     "llvm.x86.rdsspq" => "__builtin_ia32_rdsspq",
     "llvm.x86.rdtsc" => "__builtin_ia32_rdtsc",
@@ -5606,6 +7905,8 @@ match name {
     "llvm.x86.tdpbusd.internal" => "__builtin_ia32_tdpbusd_internal",
     "llvm.x86.tdpbuud" => "__builtin_ia32_tdpbuud",
     "llvm.x86.tdpbuud.internal" => "__builtin_ia32_tdpbuud_internal",
+    "llvm.x86.tdpfp16ps" => "__builtin_ia32_tdpfp16ps",
+    "llvm.x86.tdpfp16ps.internal" => "__builtin_ia32_tdpfp16ps_internal",
     "llvm.x86.testui" => "__builtin_ia32_testui",
     "llvm.x86.tileloadd64" => "__builtin_ia32_tileloadd64",
     "llvm.x86.tileloadd64.internal" => "__builtin_ia32_tileloadd64_internal",
@@ -5619,6 +7920,20 @@ match name {
     "llvm.x86.tpause" => "__builtin_ia32_tpause",
     "llvm.x86.umonitor" => "__builtin_ia32_umonitor",
     "llvm.x86.umwait" => "__builtin_ia32_umwait",
+    "llvm.x86.vbcstnebf162ps128" => "__builtin_ia32_vbcstnebf162ps128",
+    "llvm.x86.vbcstnebf162ps256" => "__builtin_ia32_vbcstnebf162ps256",
+    "llvm.x86.vbcstnesh2ps128" => "__builtin_ia32_vbcstnesh2ps128",
+    "llvm.x86.vbcstnesh2ps256" => "__builtin_ia32_vbcstnesh2ps256",
+    "llvm.x86.vcvtneebf162ps128" => "__builtin_ia32_vcvtneebf162ps128",
+    "llvm.x86.vcvtneebf162ps256" => "__builtin_ia32_vcvtneebf162ps256",
+    "llvm.x86.vcvtneeph2ps128" => "__builtin_ia32_vcvtneeph2ps128",
+    "llvm.x86.vcvtneeph2ps256" => "__builtin_ia32_vcvtneeph2ps256",
+    "llvm.x86.vcvtneobf162ps128" => "__builtin_ia32_vcvtneobf162ps128",
+    "llvm.x86.vcvtneobf162ps256" => "__builtin_ia32_vcvtneobf162ps256",
+    "llvm.x86.vcvtneoph2ps128" => "__builtin_ia32_vcvtneoph2ps128",
+    "llvm.x86.vcvtneoph2ps256" => "__builtin_ia32_vcvtneoph2ps256",
+    "llvm.x86.vcvtneps2bf16128" => "__builtin_ia32_vcvtneps2bf16128",
+    "llvm.x86.vcvtneps2bf16256" => "__builtin_ia32_vcvtneps2bf16256",
     "llvm.x86.vcvtph2ps.128" => "__builtin_ia32_vcvtph2ps",
     "llvm.x86.vcvtph2ps.256" => "__builtin_ia32_vcvtph2ps256",
     "llvm.x86.vcvtps2ph.128" => "__builtin_ia32_vcvtps2ph",
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs b/compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs
index 1b089f08f..0edec566b 100644
--- a/compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs
@@ -1,159 +1,387 @@
 use std::borrow::Cow;
 
-use gccjit::{Function, FunctionPtrType, RValue, ToRValue};
+use gccjit::{Function, FunctionPtrType, RValue, ToRValue, UnaryOp};
+use rustc_codegen_ssa::traits::BuilderMethods;
 
 use crate::{context::CodegenCx, builder::Builder};
 
-pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(builder: &Builder<'a, 'gcc, 'tcx>, gcc_func: FunctionPtrType<'gcc>, mut args: Cow<'b, [RValue<'gcc>]>, func_name: &str) -> Cow<'b, [RValue<'gcc>]> {
+pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(builder: &Builder<'a, 'gcc, 'tcx>, gcc_func: FunctionPtrType<'gcc>, mut args: Cow<'b, [RValue<'gcc>]>, func_name: &str, original_function_name: Option<&String>) -> Cow<'b, [RValue<'gcc>]> {
     // Some LLVM intrinsics do not map 1-to-1 to GCC intrinsics, so we add the missing
     // arguments here.
     if gcc_func.get_param_count() != args.len() {
         match &*func_name {
-            "__builtin_ia32_pmuldq512_mask" | "__builtin_ia32_pmuludq512_mask"
-                // FIXME(antoyo): the following intrinsics has 4 (or 5) arguments according to the doc, but is defined with 2 (or 3) arguments in library/stdarch/crates/core_arch/src/x86/avx512f.rs.
+            // NOTE: the following intrinsics have a different number of parameters in LLVM and GCC.
+            "__builtin_ia32_prold512_mask" | "__builtin_ia32_pmuldq512_mask" | "__builtin_ia32_pmuludq512_mask"
                 | "__builtin_ia32_pmaxsd512_mask" | "__builtin_ia32_pmaxsq512_mask" | "__builtin_ia32_pmaxsq256_mask"
-                | "__builtin_ia32_pmaxsq128_mask" | "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
-                | "__builtin_ia32_pmaxud512_mask" | "__builtin_ia32_pmaxuq512_mask" | "__builtin_ia32_pmaxuq256_mask"
-                | "__builtin_ia32_pmaxuq128_mask"
+                | "__builtin_ia32_pmaxsq128_mask" | "__builtin_ia32_pmaxud512_mask" | "__builtin_ia32_pmaxuq512_mask"
                 | "__builtin_ia32_pminsd512_mask" | "__builtin_ia32_pminsq512_mask" | "__builtin_ia32_pminsq256_mask"
-                | "__builtin_ia32_pminsq128_mask" | "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask"
-                | "__builtin_ia32_pminud512_mask" | "__builtin_ia32_pminuq512_mask" | "__builtin_ia32_pminuq256_mask"
-                | "__builtin_ia32_pminuq128_mask" | "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask"
+                | "__builtin_ia32_pminsq128_mask" | "__builtin_ia32_pminud512_mask" | "__builtin_ia32_pminuq512_mask"
+                | "__builtin_ia32_prolq512_mask" | "__builtin_ia32_prorq512_mask" | "__builtin_ia32_pslldi512_mask"
+                | "__builtin_ia32_psrldi512_mask" | "__builtin_ia32_psllqi512_mask" | "__builtin_ia32_psrlqi512_mask"
+                | "__builtin_ia32_pslld512_mask" | "__builtin_ia32_psrld512_mask" | "__builtin_ia32_psllq512_mask"
+                | "__builtin_ia32_psrlq512_mask" | "__builtin_ia32_psrad512_mask" | "__builtin_ia32_psraq512_mask"
+                | "__builtin_ia32_psradi512_mask" | "__builtin_ia32_psraqi512_mask" | "__builtin_ia32_psrav16si_mask"
+                | "__builtin_ia32_psrav8di_mask" | "__builtin_ia32_prolvd512_mask" | "__builtin_ia32_prorvd512_mask"
+                | "__builtin_ia32_prolvq512_mask" | "__builtin_ia32_prorvq512_mask" | "__builtin_ia32_psllv16si_mask"
+                | "__builtin_ia32_psrlv16si_mask" | "__builtin_ia32_psllv8di_mask" | "__builtin_ia32_psrlv8di_mask"
+                | "__builtin_ia32_permvarsi512_mask" | "__builtin_ia32_vpermilvarps512_mask"
+                | "__builtin_ia32_vpermilvarpd512_mask" | "__builtin_ia32_permvardi512_mask"
+                | "__builtin_ia32_permvarsf512_mask" | "__builtin_ia32_permvarqi512_mask"
+                | "__builtin_ia32_permvarqi256_mask" | "__builtin_ia32_permvarqi128_mask"
+                | "__builtin_ia32_vpmultishiftqb512_mask" | "__builtin_ia32_vpmultishiftqb256_mask"
+                | "__builtin_ia32_vpmultishiftqb128_mask"
                 => {
-                    // TODO: refactor by separating those intrinsics outside of this branch.
-                    let add_before_last_arg =
-                        match &*func_name {
-                            "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
-                                | "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask"
-                                | "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => true,
-                            _ => false,
-                        };
-                    let new_first_arg_is_zero =
-                        match &*func_name {
-                            "__builtin_ia32_pmaxuq256_mask" | "__builtin_ia32_pmaxuq128_mask"
-                                | "__builtin_ia32_pminuq256_mask" | "__builtin_ia32_pminuq128_mask" => true,
-                            _ => false
-                        };
-                    let arg3_index =
-                        match &*func_name {
-                            "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => 1,
-                            _ => 2,
-                        };
-                    let mut new_args = args.to_vec();
-                    let arg3_type = gcc_func.get_param_type(arg3_index);
-                    let first_arg =
-                        if new_first_arg_is_zero {
-                            let vector_type = arg3_type.dyncast_vector().expect("vector type");
-                            let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
-                            let num_units = vector_type.get_num_units();
-                            builder.context.new_rvalue_from_vector(None, arg3_type, &vec![zero; num_units])
-                        }
-                        else {
-                            builder.current_func().new_local(None, arg3_type, "undefined_for_intrinsic").to_rvalue()
-                        };
-                    if add_before_last_arg {
-                        new_args.insert(new_args.len() - 1, first_arg);
-                    }
-                    else {
-                        new_args.push(first_arg);
-                    }
-                    let arg4_index =
-                        match &*func_name {
-                            "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => 2,
-                            _ => 3,
-                        };
-                    let arg4_type = gcc_func.get_param_type(arg4_index);
-                    let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
-                    if add_before_last_arg {
-                        new_args.insert(new_args.len() - 1, minus_one);
-                    }
-                    else {
-                        new_args.push(minus_one);
+                let mut new_args = args.to_vec();
+                let arg3_type = gcc_func.get_param_type(2);
+                let first_arg = builder.current_func().new_local(None, arg3_type, "undefined_for_intrinsic").to_rvalue();
+                new_args.push(first_arg);
+                let arg4_type = gcc_func.get_param_type(3);
+                let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
+                new_args.push(minus_one);
+                args = new_args.into();
+            },
+            "__builtin_ia32_pmaxuq256_mask" | "__builtin_ia32_pmaxuq128_mask" | "__builtin_ia32_pminuq256_mask"
+                | "__builtin_ia32_pminuq128_mask" | "__builtin_ia32_prold256_mask" | "__builtin_ia32_prold128_mask"
+                | "__builtin_ia32_prord512_mask" | "__builtin_ia32_prord256_mask" | "__builtin_ia32_prord128_mask"
+                | "__builtin_ia32_prolq256_mask" | "__builtin_ia32_prolq128_mask" | "__builtin_ia32_prorq256_mask"
+                | "__builtin_ia32_prorq128_mask" | "__builtin_ia32_psraq256_mask" | "__builtin_ia32_psraq128_mask"
+                | "__builtin_ia32_psraqi256_mask" | "__builtin_ia32_psraqi128_mask" | "__builtin_ia32_psravq256_mask"
+                | "__builtin_ia32_psravq128_mask" | "__builtin_ia32_prolvd256_mask" | "__builtin_ia32_prolvd128_mask"
+                | "__builtin_ia32_prorvd256_mask" | "__builtin_ia32_prorvd128_mask" | "__builtin_ia32_prolvq256_mask"
+                | "__builtin_ia32_prolvq128_mask" | "__builtin_ia32_prorvq256_mask" | "__builtin_ia32_prorvq128_mask"
+                | "__builtin_ia32_permvardi256_mask" | "__builtin_ia32_permvardf512_mask" | "__builtin_ia32_permvardf256_mask"
+                | "__builtin_ia32_pmulhuw512_mask" | "__builtin_ia32_pmulhw512_mask" | "__builtin_ia32_pmulhrsw512_mask"
+                | "__builtin_ia32_pmaxuw512_mask" | "__builtin_ia32_pmaxub512_mask" | "__builtin_ia32_pmaxsw512_mask"
+                | "__builtin_ia32_pmaxsb512_mask" | "__builtin_ia32_pminuw512_mask" | "__builtin_ia32_pminub512_mask"
+                | "__builtin_ia32_pminsw512_mask" | "__builtin_ia32_pminsb512_mask"
+                | "__builtin_ia32_pmaddwd512_mask" | "__builtin_ia32_pmaddubsw512_mask" | "__builtin_ia32_packssdw512_mask"
+                | "__builtin_ia32_packsswb512_mask" | "__builtin_ia32_packusdw512_mask" | "__builtin_ia32_packuswb512_mask"
+                | "__builtin_ia32_pavgw512_mask" | "__builtin_ia32_pavgb512_mask" | "__builtin_ia32_psllw512_mask"
+                | "__builtin_ia32_psllwi512_mask" | "__builtin_ia32_psllv32hi_mask" | "__builtin_ia32_psrlw512_mask"
+                | "__builtin_ia32_psrlwi512_mask" | "__builtin_ia32_psllv16hi_mask" | "__builtin_ia32_psllv8hi_mask"
+                | "__builtin_ia32_psrlv32hi_mask" | "__builtin_ia32_psraw512_mask" | "__builtin_ia32_psrawi512_mask"
+                | "__builtin_ia32_psrlv16hi_mask" | "__builtin_ia32_psrlv8hi_mask" | "__builtin_ia32_psrav32hi_mask"
+                | "__builtin_ia32_permvarhi512_mask" | "__builtin_ia32_pshufb512_mask" | "__builtin_ia32_psrav16hi_mask"
+                | "__builtin_ia32_psrav8hi_mask" | "__builtin_ia32_permvarhi256_mask" | "__builtin_ia32_permvarhi128_mask"
+                => {
+                let mut new_args = args.to_vec();
+                let arg3_type = gcc_func.get_param_type(2);
+                let vector_type = arg3_type.dyncast_vector().expect("vector type");
+                let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
+                let num_units = vector_type.get_num_units();
+                let first_arg = builder.context.new_rvalue_from_vector(None, arg3_type, &vec![zero; num_units]);
+                new_args.push(first_arg);
+                let arg4_type = gcc_func.get_param_type(3);
+                let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
+                new_args.push(minus_one);
+                args = new_args.into();
+            },
+            "__builtin_ia32_dbpsadbw512_mask" | "__builtin_ia32_dbpsadbw256_mask" | "__builtin_ia32_dbpsadbw128_mask" => {
+                let mut new_args = args.to_vec();
+                let arg4_type = gcc_func.get_param_type(3);
+                let vector_type = arg4_type.dyncast_vector().expect("vector type");
+                let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
+                let num_units = vector_type.get_num_units();
+                let first_arg = builder.context.new_rvalue_from_vector(None, arg4_type, &vec![zero; num_units]);
+                new_args.push(first_arg);
+                let arg5_type = gcc_func.get_param_type(4);
+                let minus_one = builder.context.new_rvalue_from_int(arg5_type, -1);
+                new_args.push(minus_one);
+                args = new_args.into();
+            },
+            "__builtin_ia32_vplzcntd_512_mask" | "__builtin_ia32_vplzcntd_256_mask" | "__builtin_ia32_vplzcntd_128_mask"
+                | "__builtin_ia32_vplzcntq_512_mask" | "__builtin_ia32_vplzcntq_256_mask" | "__builtin_ia32_vplzcntq_128_mask" => {
+                let mut new_args = args.to_vec();
+                // Remove last arg as it doesn't seem to be used in GCC and is always false.
+                new_args.pop();
+                let arg2_type = gcc_func.get_param_type(1);
+                let vector_type = arg2_type.dyncast_vector().expect("vector type");
+                let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
+                let num_units = vector_type.get_num_units();
+                let first_arg = builder.context.new_rvalue_from_vector(None, arg2_type, &vec![zero; num_units]);
+                new_args.push(first_arg);
+                let arg3_type = gcc_func.get_param_type(2);
+                let minus_one = builder.context.new_rvalue_from_int(arg3_type, -1);
+                new_args.push(minus_one);
+                args = new_args.into();
+            },
+            "__builtin_ia32_vpconflictsi_512_mask" | "__builtin_ia32_vpconflictsi_256_mask"
+                | "__builtin_ia32_vpconflictsi_128_mask" | "__builtin_ia32_vpconflictdi_512_mask"
+                | "__builtin_ia32_vpconflictdi_256_mask" | "__builtin_ia32_vpconflictdi_128_mask" => {
+                let mut new_args = args.to_vec();
+                let arg2_type = gcc_func.get_param_type(1);
+                let vector_type = arg2_type.dyncast_vector().expect("vector type");
+                let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
+                let num_units = vector_type.get_num_units();
+                let first_arg = builder.context.new_rvalue_from_vector(None, arg2_type, &vec![zero; num_units]);
+                new_args.push(first_arg);
+                let arg3_type = gcc_func.get_param_type(2);
+                let minus_one = builder.context.new_rvalue_from_int(arg3_type, -1);
+                new_args.push(minus_one);
+                args = new_args.into();
+            },
+            "__builtin_ia32_pternlogd512_mask" | "__builtin_ia32_pternlogd256_mask"
+                | "__builtin_ia32_pternlogd128_mask" | "__builtin_ia32_pternlogq512_mask"
+                | "__builtin_ia32_pternlogq256_mask" | "__builtin_ia32_pternlogq128_mask" => {
+                let mut new_args = args.to_vec();
+                let arg5_type = gcc_func.get_param_type(4);
+                let minus_one = builder.context.new_rvalue_from_int(arg5_type, -1);
+                new_args.push(minus_one);
+                args = new_args.into();
+            },
+            "__builtin_ia32_vfmaddps512_mask" | "__builtin_ia32_vfmaddpd512_mask" => {
+                let mut new_args = args.to_vec();
+
+                let mut last_arg = None;
+                if args.len() == 4 {
+                    last_arg = new_args.pop();
+                }
+
+                let arg4_type = gcc_func.get_param_type(3);
+                let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
+                new_args.push(minus_one);
+
+                if args.len() == 3 {
+                    // Both llvm.fma.v16f32 and llvm.x86.avx512.vfmadd.ps.512 maps to
+                    // the same GCC intrinsic, but the former has 3 parameters and the
+                    // latter has 4 so it doesn't require this additional argument.
+                    let arg5_type = gcc_func.get_param_type(4);
+                    new_args.push(builder.context.new_rvalue_from_int(arg5_type, 4));
+                }
+
+                if let Some(last_arg) = last_arg {
+                    new_args.push(last_arg);
+                }
+
+                args = new_args.into();
+            },
+            "__builtin_ia32_addps512_mask" | "__builtin_ia32_addpd512_mask"
+                | "__builtin_ia32_subps512_mask" | "__builtin_ia32_subpd512_mask"
+                | "__builtin_ia32_mulps512_mask" | "__builtin_ia32_mulpd512_mask"
+                | "__builtin_ia32_divps512_mask" | "__builtin_ia32_divpd512_mask"
+                | "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
+                |  "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask" => {
+                let mut new_args = args.to_vec();
+                let last_arg = new_args.pop().expect("last arg");
+                let arg3_type = gcc_func.get_param_type(2);
+                let undefined = builder.current_func().new_local(None, arg3_type, "undefined_for_intrinsic").to_rvalue();
+                new_args.push(undefined);
+                let arg4_type = gcc_func.get_param_type(3);
+                let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
+                new_args.push(minus_one);
+                new_args.push(last_arg);
+                args = new_args.into();
+            },
+            "__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
+                let mut new_args = args.to_vec();
+                let last_arg = new_args.pop().expect("last arg");
+                let arg4_type = gcc_func.get_param_type(3);
+                let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
+                new_args.push(minus_one);
+                new_args.push(last_arg);
+                args = new_args.into();
+            },
+            "__builtin_ia32_vpermi2vard512_mask" | "__builtin_ia32_vpermi2vard256_mask"
+                | "__builtin_ia32_vpermi2vard128_mask" | "__builtin_ia32_vpermi2varq512_mask"
+                | "__builtin_ia32_vpermi2varq256_mask" | "__builtin_ia32_vpermi2varq128_mask"
+                | "__builtin_ia32_vpermi2varps512_mask" | "__builtin_ia32_vpermi2varps256_mask"
+                | "__builtin_ia32_vpermi2varps128_mask" | "__builtin_ia32_vpermi2varpd512_mask"
+                | "__builtin_ia32_vpermi2varpd256_mask" | "__builtin_ia32_vpermi2varpd128_mask" | "__builtin_ia32_vpmadd52huq512_mask"
+                | "__builtin_ia32_vpmadd52luq512_mask" | "__builtin_ia32_vpmadd52huq256_mask" | "__builtin_ia32_vpmadd52luq256_mask"
+                | "__builtin_ia32_vpmadd52huq128_mask"
+                => {
+                let mut new_args = args.to_vec();
+                let arg4_type = gcc_func.get_param_type(3);
+                let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
+                new_args.push(minus_one);
+                args = new_args.into();
+            },
+            "__builtin_ia32_cvtdq2ps512_mask" | "__builtin_ia32_cvtudq2ps512_mask"
+                | "__builtin_ia32_sqrtps512_mask" | "__builtin_ia32_sqrtpd512_mask" => {
+                let mut new_args = args.to_vec();
+                let last_arg = new_args.pop().expect("last arg");
+                let arg2_type = gcc_func.get_param_type(1);
+                let undefined = builder.current_func().new_local(None, arg2_type, "undefined_for_intrinsic").to_rvalue();
+                new_args.push(undefined);
+                let arg3_type = gcc_func.get_param_type(2);
+                let minus_one = builder.context.new_rvalue_from_int(arg3_type, -1);
+                new_args.push(minus_one);
+                new_args.push(last_arg);
+                args = new_args.into();
+            },
+            "__builtin_ia32_stmxcsr" => {
+                args = vec![].into();
+            },
+            "__builtin_ia32_addcarryx_u64" | "__builtin_ia32_sbb_u64" | "__builtin_ia32_addcarryx_u32" | "__builtin_ia32_sbb_u32" => {
+                let mut new_args = args.to_vec();
+                let arg2_type = gcc_func.get_param_type(1);
+                let variable = builder.current_func().new_local(None, arg2_type, "addcarryResult");
+                new_args.push(variable.get_address(None));
+                args = new_args.into();
+            },
+            "__builtin_ia32_vpermt2varqi512_mask" | "__builtin_ia32_vpermt2varqi256_mask"
+                | "__builtin_ia32_vpermt2varqi128_mask" | "__builtin_ia32_vpermt2varhi512_mask"
+                | "__builtin_ia32_vpermt2varhi256_mask" | "__builtin_ia32_vpermt2varhi128_mask"
+                => {
+                let new_args = args.to_vec();
+                let arg4_type = gcc_func.get_param_type(3);
+                let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
+                args = vec![new_args[1], new_args[0], new_args[2], minus_one].into();
+            },
+            "__builtin_ia32_xrstor" | "__builtin_ia32_xsavec" => {
+                let new_args = args.to_vec();
+                let thirty_two = builder.context.new_rvalue_from_int(new_args[1].get_type(), 32);
+                let arg2 = new_args[1] << thirty_two | new_args[2];
+                let arg2_type = gcc_func.get_param_type(1);
+                let arg2 = builder.context.new_cast(None, arg2, arg2_type);
+                args = vec![new_args[0], arg2].into();
+            },
+            "__builtin_prefetch" => {
+                let mut new_args = args.to_vec();
+                new_args.pop();
+                args = new_args.into();
+            },
+            _ => (),
+        }
+    }
+    else {
+        match &*func_name {
+            "__builtin_ia32_rndscaless_mask_round" | "__builtin_ia32_rndscalesd_mask_round" => {
+                let new_args = args.to_vec();
+                let arg3_type = gcc_func.get_param_type(2);
+                let arg3 = builder.context.new_cast(None, new_args[4], arg3_type);
+                let arg4_type = gcc_func.get_param_type(3);
+                let arg4 = builder.context.new_bitcast(None, new_args[2], arg4_type);
+                args = vec![new_args[0], new_args[1], arg3, arg4, new_args[3], new_args[5]].into();
+            },
+            // NOTE: the LLVM intrinsic receives 3 floats, but the GCC builtin requires 3 vectors.
+            // FIXME: the intrinsics like _mm_mask_fmadd_sd should probably directly call the GCC
+            // instrinsic to avoid this.
+            "__builtin_ia32_vfmaddss3_round" => {
+                let new_args = args.to_vec();
+                let arg1_type = gcc_func.get_param_type(0);
+                let arg2_type = gcc_func.get_param_type(1);
+                let arg3_type = gcc_func.get_param_type(2);
+                let a = builder.context.new_rvalue_from_vector(None, arg1_type, &[new_args[0]; 4]);
+                let b = builder.context.new_rvalue_from_vector(None, arg2_type, &[new_args[1]; 4]);
+                let c = builder.context.new_rvalue_from_vector(None, arg3_type, &[new_args[2]; 4]);
+                args = vec![a, b, c, new_args[3]].into();
+            },
+            "__builtin_ia32_vfmaddsd3_round" => {
+                let new_args = args.to_vec();
+                let arg1_type = gcc_func.get_param_type(0);
+                let arg2_type = gcc_func.get_param_type(1);
+                let arg3_type = gcc_func.get_param_type(2);
+                let a = builder.context.new_rvalue_from_vector(None, arg1_type, &[new_args[0]; 2]);
+                let b = builder.context.new_rvalue_from_vector(None, arg2_type, &[new_args[1]; 2]);
+                let c = builder.context.new_rvalue_from_vector(None, arg3_type, &[new_args[2]; 2]);
+                args = vec![a, b, c, new_args[3]].into();
+            },
+            "__builtin_ia32_vfmaddsubpd256" | "__builtin_ia32_vfmaddsubps" | "__builtin_ia32_vfmaddsubps256"
+                | "__builtin_ia32_vfmaddsubpd" => {
+                if let Some(original_function_name) = original_function_name {
+                    match &**original_function_name {
+                        "llvm.x86.fma.vfmsubadd.pd.256" | "llvm.x86.fma.vfmsubadd.ps" | "llvm.x86.fma.vfmsubadd.ps.256"
+                            | "llvm.x86.fma.vfmsubadd.pd" => {
+                            // NOTE: since both llvm.x86.fma.vfmsubadd.ps and llvm.x86.fma.vfmaddsub.ps maps to
+                            // __builtin_ia32_vfmaddsubps, only add minus if this comes from a
+                            // subadd LLVM intrinsic, e.g. _mm256_fmsubadd_pd.
+                            let mut new_args = args.to_vec();
+                            let arg3 = &mut new_args[2];
+                            *arg3 = builder.context.new_unary_op(None, UnaryOp::Minus, arg3.get_type(), *arg3);
+                            args = new_args.into();
+                        },
+                        _ => (),
                     }
-                    args = new_args.into();
-                },
-                "__builtin_ia32_pternlogd512_mask" | "__builtin_ia32_pternlogd256_mask"
-                    | "__builtin_ia32_pternlogd128_mask" | "__builtin_ia32_pternlogq512_mask"
-                    | "__builtin_ia32_pternlogq256_mask" | "__builtin_ia32_pternlogq128_mask" => {
-                        let mut new_args = args.to_vec();
-                        let arg5_type = gcc_func.get_param_type(4);
-                        let minus_one = builder.context.new_rvalue_from_int(arg5_type, -1);
-                        new_args.push(minus_one);
-                        args = new_args.into();
-                    },
-                    "__builtin_ia32_vfmaddps512_mask" | "__builtin_ia32_vfmaddpd512_mask" => {
-                        let mut new_args = args.to_vec();
-
-                        let mut last_arg = None;
-                        if args.len() == 4 {
-                            last_arg = new_args.pop();
-                        }
-
-                        let arg4_type = gcc_func.get_param_type(3);
-                        let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
-                        new_args.push(minus_one);
-
-                        if args.len() == 3 {
-                            // Both llvm.fma.v16f32 and llvm.x86.avx512.vfmadd.ps.512 maps to
-                            // the same GCC intrinsic, but the former has 3 parameters and the
-                            // latter has 4 so it doesn't require this additional argument.
-                            let arg5_type = gcc_func.get_param_type(4);
-                            new_args.push(builder.context.new_rvalue_from_int(arg5_type, 4));
-                        }
-
-                        if let Some(last_arg) = last_arg {
-                            new_args.push(last_arg);
-                        }
-
-                        args = new_args.into();
-                    },
-                    "__builtin_ia32_addps512_mask" | "__builtin_ia32_addpd512_mask"
-                        | "__builtin_ia32_subps512_mask" | "__builtin_ia32_subpd512_mask"
-                        | "__builtin_ia32_mulps512_mask" | "__builtin_ia32_mulpd512_mask"
-                        | "__builtin_ia32_divps512_mask" | "__builtin_ia32_divpd512_mask" => {
-                        let mut new_args = args.to_vec();
-                        let last_arg = new_args.pop().expect("last arg");
-                        let arg3_type = gcc_func.get_param_type(2);
-                        let undefined = builder.current_func().new_local(None, arg3_type, "undefined_for_intrinsic").to_rvalue();
-                        new_args.push(undefined);
-                        let arg4_type = gcc_func.get_param_type(3);
-                        let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
-                        new_args.push(minus_one);
-                        new_args.push(last_arg);
-                        args = new_args.into();
-                    },
-                    "__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
-                        let mut new_args = args.to_vec();
-                        let last_arg = new_args.pop().expect("last arg");
-                        let arg4_type = gcc_func.get_param_type(3);
-                        let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
-                        new_args.push(minus_one);
-                        new_args.push(last_arg);
-                        args = new_args.into();
-                    },
-                    _ => (),
+                }
+            },
+            "__builtin_ia32_ldmxcsr" => {
+                // The builtin __builtin_ia32_ldmxcsr takes an integer value while llvm.x86.sse.ldmxcsr takes a pointer,
+                // so dereference the pointer.
+                let mut new_args = args.to_vec();
+                let uint_ptr_type = builder.uint_type.make_pointer();
+                let arg1 = builder.context.new_cast(None, args[0], uint_ptr_type);
+                new_args[0] = arg1.dereference(None).to_rvalue();
+                args = new_args.into();
+            },
+            "__builtin_ia32_rcp14sd_mask" | "__builtin_ia32_rcp14ss_mask" | "__builtin_ia32_rsqrt14sd_mask"
+                | "__builtin_ia32_rsqrt14ss_mask" => {
+                let new_args = args.to_vec();
+                args = vec![new_args[1], new_args[0], new_args[2], new_args[3]].into();
+            },
+            "__builtin_ia32_sqrtsd_mask_round" | "__builtin_ia32_sqrtss_mask_round" => {
+                let new_args = args.to_vec();
+                args = vec![new_args[1], new_args[0], new_args[2], new_args[3], new_args[4]].into();
+            },
+            _ => (),
         }
     }
 
     args
 }
 
+pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(builder: &Builder<'a, 'gcc, 'tcx>, mut return_value: RValue<'gcc>, func_name: &str, args: &[RValue<'gcc>], args_adjusted: bool, orig_args: &[RValue<'gcc>]) -> RValue<'gcc> {
+    match func_name {
+        "__builtin_ia32_vfmaddss3_round" | "__builtin_ia32_vfmaddsd3_round" => {
+            #[cfg(feature="master")]
+            {
+                let zero = builder.context.new_rvalue_zero(builder.int_type);
+                return_value = builder.context.new_vector_access(None, return_value, zero).to_rvalue();
+            }
+        },
+        "__builtin_ia32_addcarryx_u64" | "__builtin_ia32_sbb_u64" | "__builtin_ia32_addcarryx_u32" | "__builtin_ia32_sbb_u32" => {
+            // Both llvm.x86.addcarry.32 and llvm.x86.addcarryx.u32 points to the same GCC builtin,
+            // but only the former requires adjusting the return value.
+            // Those 2 LLVM intrinsics differ by their argument count, that's why we check if the
+            // arguments were adjusted.
+            if args_adjusted {
+                let last_arg = args.last().expect("last arg");
+                let field1 = builder.context.new_field(None, builder.u8_type, "carryFlag");
+                let field2 = builder.context.new_field(None, args[1].get_type(), "carryResult");
+                let struct_type = builder.context.new_struct_type(None, "addcarryResult", &[field1, field2]);
+                return_value = builder.context.new_struct_constructor(None, struct_type.as_type(), None, &[return_value, last_arg.dereference(None).to_rvalue()]);
+            }
+        },
+        "__builtin_ia32_stmxcsr" => {
+            // The builtin __builtin_ia32_stmxcsr returns a value while llvm.x86.sse.stmxcsr writes
+            // the result in its pointer argument.
+            // We removed the argument since __builtin_ia32_stmxcsr takes no arguments, so we need
+            // to get back the original argument to get the pointer we need to write the result to.
+            let uint_ptr_type = builder.uint_type.make_pointer();
+            let ptr = builder.context.new_cast(None, orig_args[0], uint_ptr_type);
+            builder.llbb().add_assignment(None, ptr.dereference(None), return_value);
+            // The return value was assigned to the result pointer above. In order to not call the
+            // builtin twice, we overwrite the return value with a dummy value.
+            return_value = builder.context.new_rvalue_zero(builder.int_type);
+        },
+        _ => (),
+    }
+
+    return_value
+}
+
 pub fn ignore_arg_cast(func_name: &str, index: usize, args_len: usize) -> bool {
-    // NOTE: these intrinsics have missing parameters before the last one, so ignore the
-    // last argument type check.
     // FIXME(antoyo): find a way to refactor in order to avoid this hack.
     match func_name {
+        // NOTE: these intrinsics have missing parameters before the last one, so ignore the
+        // last argument type check.
         "__builtin_ia32_maxps512_mask" | "__builtin_ia32_maxpd512_mask"
             | "__builtin_ia32_minps512_mask" | "__builtin_ia32_minpd512_mask" | "__builtin_ia32_sqrtps512_mask"
             | "__builtin_ia32_sqrtpd512_mask" | "__builtin_ia32_addps512_mask" | "__builtin_ia32_addpd512_mask"
             | "__builtin_ia32_subps512_mask" | "__builtin_ia32_subpd512_mask"
             | "__builtin_ia32_mulps512_mask" | "__builtin_ia32_mulpd512_mask"
             | "__builtin_ia32_divps512_mask" | "__builtin_ia32_divpd512_mask"
-            | "__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
+            | "__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask"
+            | "__builtin_ia32_cvtdq2ps512_mask" | "__builtin_ia32_cvtudq2ps512_mask" => {
                 if index == args_len - 1 {
                     return true;
                 }
             },
+        "__builtin_ia32_rndscaless_mask_round" | "__builtin_ia32_rndscalesd_mask_round" => {
+            if index == 2 || index == 3 {
+                return true;
+            }
+        },
         "__builtin_ia32_vfmaddps512_mask" | "__builtin_ia32_vfmaddpd512_mask" => {
             // Since there are two LLVM intrinsics that map to each of these GCC builtins and only
             // one of them has a missing parameter before the last one, we check the number of
@@ -162,6 +390,14 @@ pub fn ignore_arg_cast(func_name: &str, index: usize, args_len: usize) -> bool {
                 return true;
             }
         },
+        // NOTE: the LLVM intrinsic receives 3 floats, but the GCC builtin requires 3 vectors.
+        "__builtin_ia32_vfmaddss3_round" | "__builtin_ia32_vfmaddsd3_round" => return true,
+        "__builtin_ia32_vplzcntd_512_mask" | "__builtin_ia32_vplzcntd_256_mask" | "__builtin_ia32_vplzcntd_128_mask"
+            | "__builtin_ia32_vplzcntq_512_mask" | "__builtin_ia32_vplzcntq_256_mask" | "__builtin_ia32_vplzcntq_128_mask" => {
+            if index == args_len - 1 {
+                return true;
+            }
+        },
         _ => (),
     }
 
@@ -171,7 +407,7 @@ pub fn ignore_arg_cast(func_name: &str, index: usize, args_len: usize) -> bool {
 #[cfg(not(feature="master"))]
 pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function<'gcc> {
     match name {
-        "llvm.x86.xgetbv" => {
+        "llvm.x86.xgetbv" | "llvm.x86.sse2.pause" => {
             let gcc_name = "__builtin_trap";
             let func = cx.context.get_builtin_function(gcc_name);
             cx.functions.borrow_mut().insert(gcc_name.to_string(), func);
@@ -183,24 +419,26 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
 
 #[cfg(feature="master")]
 pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function<'gcc> {
+    match name {
+        "llvm.prefetch" => {
+            let gcc_name = "__builtin_prefetch";
+            let func = cx.context.get_builtin_function(gcc_name);
+            cx.functions.borrow_mut().insert(gcc_name.to_string(), func);
+            return func
+        },
+        _ => (),
+    }
+
     let gcc_name = match name {
         "llvm.x86.xgetbv" => "__builtin_ia32_xgetbv",
         // NOTE: this doc specifies the equivalent GCC builtins: http://huonw.github.io/llvmint/llvmint/x86/index.html
         "llvm.sqrt.v2f64" => "__builtin_ia32_sqrtpd",
         "llvm.x86.avx512.pmul.dq.512" => "__builtin_ia32_pmuldq512_mask",
         "llvm.x86.avx512.pmulu.dq.512" => "__builtin_ia32_pmuludq512_mask",
-        "llvm.x86.avx512.mask.pmaxs.q.256" => "__builtin_ia32_pmaxsq256_mask",
-        "llvm.x86.avx512.mask.pmaxs.q.128" => "__builtin_ia32_pmaxsq128_mask",
         "llvm.x86.avx512.max.ps.512" => "__builtin_ia32_maxps512_mask",
         "llvm.x86.avx512.max.pd.512" => "__builtin_ia32_maxpd512_mask",
-        "llvm.x86.avx512.mask.pmaxu.q.256" => "__builtin_ia32_pmaxuq256_mask",
-        "llvm.x86.avx512.mask.pmaxu.q.128" => "__builtin_ia32_pmaxuq128_mask",
-        "llvm.x86.avx512.mask.pmins.q.256" => "__builtin_ia32_pminsq256_mask",
-        "llvm.x86.avx512.mask.pmins.q.128" => "__builtin_ia32_pminsq128_mask",
         "llvm.x86.avx512.min.ps.512" => "__builtin_ia32_minps512_mask",
         "llvm.x86.avx512.min.pd.512" => "__builtin_ia32_minpd512_mask",
-        "llvm.x86.avx512.mask.pminu.q.256" => "__builtin_ia32_pminuq256_mask",
-        "llvm.x86.avx512.mask.pminu.q.128" => "__builtin_ia32_pminuq128_mask",
         "llvm.fma.v16f32" => "__builtin_ia32_vfmaddps512_mask",
         "llvm.fma.v8f64" => "__builtin_ia32_vfmaddpd512_mask",
         "llvm.x86.avx512.vfmaddsub.ps.512" => "__builtin_ia32_vfmaddsubps512_mask",
@@ -221,6 +459,153 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
         "llvm.x86.avx512.div.pd.512" => "__builtin_ia32_divpd512_mask",
         "llvm.x86.avx512.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask",
         "llvm.x86.avx512.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask",
+        "llvm.x86.avx512.sitofp.round.v16f32.v16i32" => "__builtin_ia32_cvtdq2ps512_mask",
+        "llvm.x86.avx512.uitofp.round.v16f32.v16i32" => "__builtin_ia32_cvtudq2ps512_mask",
+        "llvm.x86.avx512.mask.ucmp.d.512" => "__builtin_ia32_ucmpd512_mask",
+        "llvm.x86.avx512.mask.ucmp.d.256" => "__builtin_ia32_ucmpd256_mask",
+        "llvm.x86.avx512.mask.ucmp.d.128" => "__builtin_ia32_ucmpd128_mask",
+        "llvm.x86.avx512.mask.cmp.d.512" => "__builtin_ia32_cmpd512_mask",
+        "llvm.x86.avx512.mask.cmp.d.256" => "__builtin_ia32_cmpd256_mask",
+        "llvm.x86.avx512.mask.cmp.d.128" => "__builtin_ia32_cmpd128_mask",
+        "llvm.x86.avx512.mask.ucmp.q.512" => "__builtin_ia32_ucmpq512_mask",
+        "llvm.x86.avx512.mask.ucmp.q.256" => "__builtin_ia32_ucmpq256_mask",
+        "llvm.x86.avx512.mask.ucmp.q.128" => "__builtin_ia32_ucmpq128_mask",
+        "llvm.x86.avx512.mask.cmp.q.512" => "__builtin_ia32_cmpq512_mask",
+        "llvm.x86.avx512.mask.cmp.q.256" => "__builtin_ia32_cmpq256_mask",
+        "llvm.x86.avx512.mask.cmp.q.128" => "__builtin_ia32_cmpq128_mask",
+        "llvm.x86.avx512.mask.max.ss.round" => "__builtin_ia32_maxss_mask_round",
+        "llvm.x86.avx512.mask.max.sd.round" => "__builtin_ia32_maxsd_mask_round",
+        "llvm.x86.avx512.mask.min.ss.round" => "__builtin_ia32_minss_mask_round",
+        "llvm.x86.avx512.mask.min.sd.round" => "__builtin_ia32_minsd_mask_round",
+        "llvm.x86.avx512.mask.sqrt.ss" => "__builtin_ia32_sqrtss_mask_round",
+        "llvm.x86.avx512.mask.sqrt.sd" => "__builtin_ia32_sqrtsd_mask_round",
+        "llvm.x86.avx512.mask.getexp.ss" => "__builtin_ia32_getexpss_mask_round",
+        "llvm.x86.avx512.mask.getexp.sd" => "__builtin_ia32_getexpsd_mask_round",
+        "llvm.x86.avx512.mask.getmant.ss" => "__builtin_ia32_getmantss_mask_round",
+        "llvm.x86.avx512.mask.getmant.sd" => "__builtin_ia32_getmantsd_mask_round",
+        "llvm.x86.avx512.mask.rndscale.ss" => "__builtin_ia32_rndscaless_mask_round",
+        "llvm.x86.avx512.mask.rndscale.sd" => "__builtin_ia32_rndscalesd_mask_round",
+        "llvm.x86.avx512.mask.scalef.ss" => "__builtin_ia32_scalefss_mask_round",
+        "llvm.x86.avx512.mask.scalef.sd" => "__builtin_ia32_scalefsd_mask_round",
+        "llvm.x86.avx512.vfmadd.f32" => "__builtin_ia32_vfmaddss3_round",
+        "llvm.x86.avx512.vfmadd.f64" => "__builtin_ia32_vfmaddsd3_round",
+        "llvm.ceil.v4f64" => "__builtin_ia32_ceilpd256",
+        "llvm.ceil.v8f32" => "__builtin_ia32_ceilps256",
+        "llvm.floor.v4f64" => "__builtin_ia32_floorpd256",
+        "llvm.floor.v8f32" => "__builtin_ia32_floorps256",
+        "llvm.sqrt.v4f64" => "__builtin_ia32_sqrtpd256",
+        "llvm.x86.sse.stmxcsr" => "__builtin_ia32_stmxcsr",
+        "llvm.x86.sse.ldmxcsr" => "__builtin_ia32_ldmxcsr",
+        "llvm.ctpop.v16i32" => "__builtin_ia32_vpopcountd_v16si",
+        "llvm.ctpop.v8i32" => "__builtin_ia32_vpopcountd_v8si",
+        "llvm.ctpop.v4i32" => "__builtin_ia32_vpopcountd_v4si",
+        "llvm.ctpop.v8i64" => "__builtin_ia32_vpopcountq_v8di",
+        "llvm.ctpop.v4i64" => "__builtin_ia32_vpopcountq_v4di",
+        "llvm.ctpop.v2i64" => "__builtin_ia32_vpopcountq_v2di",
+        "llvm.x86.addcarry.64" => "__builtin_ia32_addcarryx_u64",
+        "llvm.x86.subborrow.64" => "__builtin_ia32_sbb_u64",
+        "llvm.floor.v2f64" => "__builtin_ia32_floorpd",
+        "llvm.floor.v4f32" => "__builtin_ia32_floorps",
+        "llvm.ceil.v2f64" => "__builtin_ia32_ceilpd",
+        "llvm.ceil.v4f32" => "__builtin_ia32_ceilps",
+        "llvm.fma.v2f64" => "__builtin_ia32_vfmaddpd",
+        "llvm.fma.v4f64" => "__builtin_ia32_vfmaddpd256",
+        "llvm.fma.v4f32" => "__builtin_ia32_vfmaddps",
+        "llvm.fma.v8f32" => "__builtin_ia32_vfmaddps256",
+        "llvm.ctlz.v16i32" => "__builtin_ia32_vplzcntd_512_mask",
+        "llvm.ctlz.v8i32" => "__builtin_ia32_vplzcntd_256_mask",
+        "llvm.ctlz.v4i32" => "__builtin_ia32_vplzcntd_128_mask",
+        "llvm.ctlz.v8i64" => "__builtin_ia32_vplzcntq_512_mask",
+        "llvm.ctlz.v4i64" => "__builtin_ia32_vplzcntq_256_mask",
+        "llvm.ctlz.v2i64" => "__builtin_ia32_vplzcntq_128_mask",
+        "llvm.ctpop.v32i16" => "__builtin_ia32_vpopcountw_v32hi",
+        "llvm.x86.fma.vfmsub.sd" => "__builtin_ia32_vfmsubsd3",
+        "llvm.x86.fma.vfmsub.ss" => "__builtin_ia32_vfmsubss3",
+        "llvm.x86.fma.vfmsubadd.pd" => "__builtin_ia32_vfmaddsubpd",
+        "llvm.x86.fma.vfmsubadd.pd.256" => "__builtin_ia32_vfmaddsubpd256",
+        "llvm.x86.fma.vfmsubadd.ps" => "__builtin_ia32_vfmaddsubps",
+        "llvm.x86.fma.vfmsubadd.ps.256" => "__builtin_ia32_vfmaddsubps256",
+        "llvm.x86.fma.vfnmadd.sd" => "__builtin_ia32_vfnmaddsd3",
+        "llvm.x86.fma.vfnmadd.ss" => "__builtin_ia32_vfnmaddss3",
+        "llvm.x86.fma.vfnmsub.sd" => "__builtin_ia32_vfnmsubsd3",
+        "llvm.x86.fma.vfnmsub.ss" => "__builtin_ia32_vfnmsubss3",
+        "llvm.x86.avx512.conflict.d.512" => "__builtin_ia32_vpconflictsi_512_mask",
+        "llvm.x86.avx512.conflict.d.256" => "__builtin_ia32_vpconflictsi_256_mask",
+        "llvm.x86.avx512.conflict.d.128" => "__builtin_ia32_vpconflictsi_128_mask",
+        "llvm.x86.avx512.conflict.q.512" => "__builtin_ia32_vpconflictdi_512_mask",
+        "llvm.x86.avx512.conflict.q.256" => "__builtin_ia32_vpconflictdi_256_mask",
+        "llvm.x86.avx512.conflict.q.128" => "__builtin_ia32_vpconflictdi_128_mask",
+        "llvm.x86.avx512.vpermi2var.qi.512" => "__builtin_ia32_vpermt2varqi512_mask",
+        "llvm.x86.avx512.vpermi2var.qi.256" => "__builtin_ia32_vpermt2varqi256_mask",
+        "llvm.x86.avx512.vpermi2var.qi.128" => "__builtin_ia32_vpermt2varqi128_mask",
+        "llvm.x86.avx512.permvar.qi.512" => "__builtin_ia32_permvarqi512_mask",
+        "llvm.x86.avx512.permvar.qi.256" => "__builtin_ia32_permvarqi256_mask",
+        "llvm.x86.avx512.permvar.qi.128" => "__builtin_ia32_permvarqi128_mask",
+        "llvm.x86.avx512.pmultishift.qb.512" => "__builtin_ia32_vpmultishiftqb512_mask",
+        "llvm.x86.avx512.pmultishift.qb.256" => "__builtin_ia32_vpmultishiftqb256_mask",
+        "llvm.x86.avx512.pmultishift.qb.128" => "__builtin_ia32_vpmultishiftqb128_mask",
+        "llvm.ctpop.v16i16" => "__builtin_ia32_vpopcountw_v16hi",
+        "llvm.ctpop.v8i16" => "__builtin_ia32_vpopcountw_v8hi",
+        "llvm.ctpop.v64i8" => "__builtin_ia32_vpopcountb_v64qi",
+        "llvm.ctpop.v32i8" => "__builtin_ia32_vpopcountb_v32qi",
+        "llvm.ctpop.v16i8" => "__builtin_ia32_vpopcountb_v16qi",
+        "llvm.x86.avx512.mask.vpshufbitqmb.512" => "__builtin_ia32_vpshufbitqmb512_mask",
+        "llvm.x86.avx512.mask.vpshufbitqmb.256" => "__builtin_ia32_vpshufbitqmb256_mask",
+        "llvm.x86.avx512.mask.vpshufbitqmb.128" => "__builtin_ia32_vpshufbitqmb128_mask",
+        "llvm.x86.avx512.mask.ucmp.w.512" => "__builtin_ia32_ucmpw512_mask",
+        "llvm.x86.avx512.mask.ucmp.w.256" => "__builtin_ia32_ucmpw256_mask",
+        "llvm.x86.avx512.mask.ucmp.w.128" => "__builtin_ia32_ucmpw128_mask",
+        "llvm.x86.avx512.mask.ucmp.b.512" => "__builtin_ia32_ucmpb512_mask",
+        "llvm.x86.avx512.mask.ucmp.b.256" => "__builtin_ia32_ucmpb256_mask",
+        "llvm.x86.avx512.mask.ucmp.b.128" => "__builtin_ia32_ucmpb128_mask",
+        "llvm.x86.avx512.mask.cmp.w.512" => "__builtin_ia32_cmpw512_mask",
+        "llvm.x86.avx512.mask.cmp.w.256" => "__builtin_ia32_cmpw256_mask",
+        "llvm.x86.avx512.mask.cmp.w.128" => "__builtin_ia32_cmpw128_mask",
+        "llvm.x86.avx512.mask.cmp.b.512" => "__builtin_ia32_cmpb512_mask",
+        "llvm.x86.avx512.mask.cmp.b.256" => "__builtin_ia32_cmpb256_mask",
+        "llvm.x86.avx512.mask.cmp.b.128" => "__builtin_ia32_cmpb128_mask",
+        "llvm.x86.xrstor" => "__builtin_ia32_xrstor",
+        "llvm.x86.xsavec" => "__builtin_ia32_xsavec",
+        "llvm.x86.addcarry.32" => "__builtin_ia32_addcarryx_u32",
+        "llvm.x86.subborrow.32" => "__builtin_ia32_sbb_u32",
+        "llvm.x86.avx512.mask.compress.store.w.512" => "__builtin_ia32_compressstoreuhi512_mask",
+        "llvm.x86.avx512.mask.compress.store.w.256" => "__builtin_ia32_compressstoreuhi256_mask",
+        "llvm.x86.avx512.mask.compress.store.w.128" => "__builtin_ia32_compressstoreuhi128_mask",
+        "llvm.x86.avx512.mask.compress.store.b.512" => "__builtin_ia32_compressstoreuqi512_mask",
+        "llvm.x86.avx512.mask.compress.store.b.256" => "__builtin_ia32_compressstoreuqi256_mask",
+        "llvm.x86.avx512.mask.compress.store.b.128" => "__builtin_ia32_compressstoreuqi128_mask",
+        "llvm.x86.avx512.mask.compress.w.512" => "__builtin_ia32_compresshi512_mask",
+        "llvm.x86.avx512.mask.compress.w.256" => "__builtin_ia32_compresshi256_mask",
+        "llvm.x86.avx512.mask.compress.w.128" => "__builtin_ia32_compresshi128_mask",
+        "llvm.x86.avx512.mask.compress.b.512" => "__builtin_ia32_compressqi512_mask",
+        "llvm.x86.avx512.mask.compress.b.256" => "__builtin_ia32_compressqi256_mask",
+        "llvm.x86.avx512.mask.compress.b.128" => "__builtin_ia32_compressqi128_mask",
+        "llvm.x86.avx512.mask.expand.w.512" => "__builtin_ia32_expandhi512_mask",
+        "llvm.x86.avx512.mask.expand.w.256" => "__builtin_ia32_expandhi256_mask",
+        "llvm.x86.avx512.mask.expand.w.128" => "__builtin_ia32_expandhi128_mask",
+        "llvm.x86.avx512.mask.expand.b.512" => "__builtin_ia32_expandqi512_mask",
+        "llvm.x86.avx512.mask.expand.b.256" => "__builtin_ia32_expandqi256_mask",
+        "llvm.x86.avx512.mask.expand.b.128" => "__builtin_ia32_expandqi128_mask",
+        "llvm.fshl.v8i64" => "__builtin_ia32_vpshldv_v8di",
+        "llvm.fshl.v4i64" => "__builtin_ia32_vpshldv_v4di",
+        "llvm.fshl.v2i64" => "__builtin_ia32_vpshldv_v2di",
+        "llvm.fshl.v16i32" => "__builtin_ia32_vpshldv_v16si",
+        "llvm.fshl.v8i32" => "__builtin_ia32_vpshldv_v8si",
+        "llvm.fshl.v4i32" => "__builtin_ia32_vpshldv_v4si",
+        "llvm.fshl.v32i16" => "__builtin_ia32_vpshldv_v32hi",
+        "llvm.fshl.v16i16" => "__builtin_ia32_vpshldv_v16hi",
+        "llvm.fshl.v8i16" => "__builtin_ia32_vpshldv_v8hi",
+        "llvm.fshr.v8i64" => "__builtin_ia32_vpshrdv_v8di",
+        "llvm.fshr.v4i64" => "__builtin_ia32_vpshrdv_v4di",
+        "llvm.fshr.v2i64" => "__builtin_ia32_vpshrdv_v2di",
+        "llvm.fshr.v16i32" => "__builtin_ia32_vpshrdv_v16si",
+        "llvm.fshr.v8i32" => "__builtin_ia32_vpshrdv_v8si",
+        "llvm.fshr.v4i32" => "__builtin_ia32_vpshrdv_v4si",
+        "llvm.fshr.v32i16" => "__builtin_ia32_vpshrdv_v32hi",
+        "llvm.fshr.v16i16" => "__builtin_ia32_vpshrdv_v16hi",
+        "llvm.fshr.v8i16" => "__builtin_ia32_vpshrdv_v8hi",
+        "llvm.x86.fma.vfmadd.sd" => "__builtin_ia32_vfmaddsd3",
+        "llvm.x86.fma.vfmadd.ss" => "__builtin_ia32_vfmaddss3",
 
         // The above doc points to unknown builtins for the following, so override them:
         "llvm.x86.avx2.gather.d.d" => "__builtin_ia32_gathersiv4si",
@@ -239,7 +624,151 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
         "llvm.x86.avx2.gather.q.q.256" => "__builtin_ia32_gatherdiv4di",
         "llvm.x86.avx2.gather.q.pd" => "__builtin_ia32_gatherdiv2df",
         "llvm.x86.avx2.gather.q.pd.256" => "__builtin_ia32_gatherdiv4df",
-        "" => "",
+        "llvm.x86.avx512.pslli.d.512" => "__builtin_ia32_pslldi512_mask",
+        "llvm.x86.avx512.psrli.d.512" => "__builtin_ia32_psrldi512_mask",
+        "llvm.x86.avx512.pslli.q.512" => "__builtin_ia32_psllqi512_mask",
+        "llvm.x86.avx512.psrli.q.512" => "__builtin_ia32_psrlqi512_mask",
+        "llvm.x86.avx512.psll.d.512" => "__builtin_ia32_pslld512_mask",
+        "llvm.x86.avx512.psrl.d.512" => "__builtin_ia32_psrld512_mask",
+        "llvm.x86.avx512.psll.q.512" => "__builtin_ia32_psllq512_mask",
+        "llvm.x86.avx512.psrl.q.512" => "__builtin_ia32_psrlq512_mask",
+        "llvm.x86.avx512.psra.d.512" => "__builtin_ia32_psrad512_mask",
+        "llvm.x86.avx512.psra.q.512" => "__builtin_ia32_psraq512_mask",
+        "llvm.x86.avx512.psra.q.256" => "__builtin_ia32_psraq256_mask",
+        "llvm.x86.avx512.psra.q.128" => "__builtin_ia32_psraq128_mask",
+        "llvm.x86.avx512.psrai.d.512" => "__builtin_ia32_psradi512_mask",
+        "llvm.x86.avx512.psrai.q.512" => "__builtin_ia32_psraqi512_mask",
+        "llvm.x86.avx512.psrai.q.256" => "__builtin_ia32_psraqi256_mask",
+        "llvm.x86.avx512.psrai.q.128" => "__builtin_ia32_psraqi128_mask",
+        "llvm.x86.avx512.psrav.d.512" => "__builtin_ia32_psrav16si_mask",
+        "llvm.x86.avx512.psrav.q.512" => "__builtin_ia32_psrav8di_mask",
+        "llvm.x86.avx512.psrav.q.256" => "__builtin_ia32_psravq256_mask",
+        "llvm.x86.avx512.psrav.q.128" => "__builtin_ia32_psravq128_mask",
+        "llvm.x86.avx512.psllv.d.512" => "__builtin_ia32_psllv16si_mask",
+        "llvm.x86.avx512.psrlv.d.512" => "__builtin_ia32_psrlv16si_mask",
+        "llvm.x86.avx512.psllv.q.512" => "__builtin_ia32_psllv8di_mask",
+        "llvm.x86.avx512.psrlv.q.512" => "__builtin_ia32_psrlv8di_mask",
+        "llvm.x86.avx512.permvar.si.512" => "__builtin_ia32_permvarsi512_mask",
+        "llvm.x86.avx512.vpermilvar.ps.512" => "__builtin_ia32_vpermilvarps512_mask",
+        "llvm.x86.avx512.vpermilvar.pd.512" => "__builtin_ia32_vpermilvarpd512_mask",
+        "llvm.x86.avx512.permvar.di.512" => "__builtin_ia32_permvardi512_mask",
+        "llvm.x86.avx512.permvar.di.256" => "__builtin_ia32_permvardi256_mask",
+        "llvm.x86.avx512.permvar.sf.512" => "__builtin_ia32_permvarsf512_mask",
+        "llvm.x86.avx512.permvar.df.512" => "__builtin_ia32_permvardf512_mask",
+        "llvm.x86.avx512.permvar.df.256" => "__builtin_ia32_permvardf256_mask",
+        "llvm.x86.avx512.vpermi2var.d.512" => "__builtin_ia32_vpermi2vard512_mask",
+        "llvm.x86.avx512.vpermi2var.d.256" => "__builtin_ia32_vpermi2vard256_mask",
+        "llvm.x86.avx512.vpermi2var.d.128" => "__builtin_ia32_vpermi2vard128_mask",
+        "llvm.x86.avx512.vpermi2var.q.512" => "__builtin_ia32_vpermi2varq512_mask",
+        "llvm.x86.avx512.vpermi2var.q.256" => "__builtin_ia32_vpermi2varq256_mask",
+        "llvm.x86.avx512.vpermi2var.q.128" => "__builtin_ia32_vpermi2varq128_mask",
+        "llvm.x86.avx512.vpermi2var.ps.512" => "__builtin_ia32_vpermi2varps512_mask",
+        "llvm.x86.avx512.vpermi2var.ps.256" => "__builtin_ia32_vpermi2varps256_mask",
+        "llvm.x86.avx512.vpermi2var.ps.128" => "__builtin_ia32_vpermi2varps128_mask",
+        "llvm.x86.avx512.vpermi2var.pd.512" => "__builtin_ia32_vpermi2varpd512_mask",
+        "llvm.x86.avx512.vpermi2var.pd.256" => "__builtin_ia32_vpermi2varpd256_mask",
+        "llvm.x86.avx512.vpermi2var.pd.128" => "__builtin_ia32_vpermi2varpd128_mask",
+        "llvm.x86.avx512.mask.add.ss.round" => "__builtin_ia32_addss_mask_round",
+        "llvm.x86.avx512.mask.add.sd.round" => "__builtin_ia32_addsd_mask_round",
+        "llvm.x86.avx512.mask.sub.ss.round" => "__builtin_ia32_subss_mask_round",
+        "llvm.x86.avx512.mask.sub.sd.round" => "__builtin_ia32_subsd_mask_round",
+        "llvm.x86.avx512.mask.mul.ss.round" => "__builtin_ia32_mulss_mask_round",
+        "llvm.x86.avx512.mask.mul.sd.round" => "__builtin_ia32_mulsd_mask_round",
+        "llvm.x86.avx512.mask.div.ss.round" => "__builtin_ia32_divss_mask_round",
+        "llvm.x86.avx512.mask.div.sd.round" => "__builtin_ia32_divsd_mask_round",
+        "llvm.x86.avx512.mask.cvtss2sd.round" => "__builtin_ia32_cvtss2sd_mask_round",
+        "llvm.x86.avx512.mask.cvtsd2ss.round" => "__builtin_ia32_cvtsd2ss_mask_round",
+        "llvm.x86.avx512.mask.range.ss" => "__builtin_ia32_rangess128_mask_round",
+        "llvm.x86.avx512.mask.range.sd" => "__builtin_ia32_rangesd128_mask_round",
+        "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask_round",
+        "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask_round",
+        "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask_round",
+        "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask_round",
+        "llvm.x86.avx512fp16.mask.add.sh.round" => "__builtin_ia32_addsh_mask_round",
+        "llvm.x86.avx512fp16.mask.div.sh.round" => "__builtin_ia32_divsh_mask_round",
+        "llvm.x86.avx512fp16.mask.getmant.sh" => "__builtin_ia32_getmantsh_mask_round",
+        "llvm.x86.avx512fp16.mask.max.sh.round" => "__builtin_ia32_maxsh_mask_round",
+        "llvm.x86.avx512fp16.mask.min.sh.round" => "__builtin_ia32_minsh_mask_round",
+        "llvm.x86.avx512fp16.mask.mul.sh.round" => "__builtin_ia32_mulsh_mask_round",
+        "llvm.x86.avx512fp16.mask.rndscale.sh" => "__builtin_ia32_rndscalesh_mask_round",
+        "llvm.x86.avx512fp16.mask.scalef.sh" => "__builtin_ia32_scalefsh_mask_round",
+        "llvm.x86.avx512fp16.mask.sub.sh.round" => "__builtin_ia32_subsh_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtsd2sh.round" => "__builtin_ia32_vcvtsd2sh_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtsh2sd.round" => "__builtin_ia32_vcvtsh2sd_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtsh2ss.round" => "__builtin_ia32_vcvtsh2ss_mask_round",
+        "llvm.x86.avx512fp16.mask.vcvtss2sh.round" => "__builtin_ia32_vcvtss2sh_mask_round",
+        "llvm.x86.aesni.aesenc.256" => "__builtin_ia32_vaesenc_v32qi",
+        "llvm.x86.aesni.aesenclast.256" => "__builtin_ia32_vaesenclast_v32qi",
+        "llvm.x86.aesni.aesdec.256" => "__builtin_ia32_vaesdec_v32qi",
+        "llvm.x86.aesni.aesdeclast.256" => "__builtin_ia32_vaesdeclast_v32qi",
+        "llvm.x86.aesni.aesenc.512" => "__builtin_ia32_vaesenc_v64qi",
+        "llvm.x86.aesni.aesenclast.512" => "__builtin_ia32_vaesenclast_v64qi",
+        "llvm.x86.aesni.aesdec.512" => "__builtin_ia32_vaesdec_v64qi",
+        "llvm.x86.aesni.aesdeclast.512" => "__builtin_ia32_vaesdeclast_v64qi",
+        "llvm.x86.avx512bf16.cvtne2ps2bf16.128" => "__builtin_ia32_cvtne2ps2bf16_v8bf",
+        "llvm.x86.avx512bf16.cvtne2ps2bf16.256" => "__builtin_ia32_cvtne2ps2bf16_v16bf",
+        "llvm.x86.avx512bf16.cvtne2ps2bf16.512" => "__builtin_ia32_cvtne2ps2bf16_v32bf",
+        "llvm.x86.avx512bf16.cvtneps2bf16.256" => "__builtin_ia32_cvtneps2bf16_v8sf",
+        "llvm.x86.avx512bf16.cvtneps2bf16.512" => "__builtin_ia32_cvtneps2bf16_v16sf",
+        "llvm.x86.avx512bf16.dpbf16ps.128" => "__builtin_ia32_dpbf16ps_v4sf",
+        "llvm.x86.avx512bf16.dpbf16ps.256" => "__builtin_ia32_dpbf16ps_v8sf",
+        "llvm.x86.avx512bf16.dpbf16ps.512" => "__builtin_ia32_dpbf16ps_v16sf",
+        "llvm.x86.pclmulqdq.512" => "__builtin_ia32_vpclmulqdq_v8di",
+        "llvm.x86.pclmulqdq.256" => "__builtin_ia32_vpclmulqdq_v4di",
+        "llvm.x86.avx512.pmulhu.w.512" => "__builtin_ia32_pmulhuw512_mask",
+        "llvm.x86.avx512.pmulh.w.512" => "__builtin_ia32_pmulhw512_mask",
+        "llvm.x86.avx512.pmul.hr.sw.512" => "__builtin_ia32_pmulhrsw512_mask",
+        "llvm.x86.avx512.pmaddw.d.512" => "__builtin_ia32_pmaddwd512_mask",
+        "llvm.x86.avx512.pmaddubs.w.512" => "__builtin_ia32_pmaddubsw512_mask",
+        "llvm.x86.avx512.packssdw.512" => "__builtin_ia32_packssdw512_mask",
+        "llvm.x86.avx512.packsswb.512" => "__builtin_ia32_packsswb512_mask",
+        "llvm.x86.avx512.packusdw.512" => "__builtin_ia32_packusdw512_mask",
+        "llvm.x86.avx512.packuswb.512" => "__builtin_ia32_packuswb512_mask",
+        "llvm.x86.avx512.pavg.w.512" => "__builtin_ia32_pavgw512_mask",
+        "llvm.x86.avx512.pavg.b.512" => "__builtin_ia32_pavgb512_mask",
+        "llvm.x86.avx512.psll.w.512" => "__builtin_ia32_psllw512_mask",
+        "llvm.x86.avx512.pslli.w.512" => "__builtin_ia32_psllwi512_mask",
+        "llvm.x86.avx512.psllv.w.512" => "__builtin_ia32_psllv32hi_mask",
+        "llvm.x86.avx512.psllv.w.256" => "__builtin_ia32_psllv16hi_mask",
+        "llvm.x86.avx512.psllv.w.128" => "__builtin_ia32_psllv8hi_mask",
+        "llvm.x86.avx512.psrl.w.512" => "__builtin_ia32_psrlw512_mask",
+        "llvm.x86.avx512.psrli.w.512" => "__builtin_ia32_psrlwi512_mask",
+        "llvm.x86.avx512.psrlv.w.512" => "__builtin_ia32_psrlv32hi_mask",
+        "llvm.x86.avx512.psrlv.w.256" => "__builtin_ia32_psrlv16hi_mask",
+        "llvm.x86.avx512.psrlv.w.128" => "__builtin_ia32_psrlv8hi_mask",
+        "llvm.x86.avx512.psra.w.512" => "__builtin_ia32_psraw512_mask",
+        "llvm.x86.avx512.psrai.w.512" => "__builtin_ia32_psrawi512_mask",
+        "llvm.x86.avx512.psrav.w.512" => "__builtin_ia32_psrav32hi_mask",
+        "llvm.x86.avx512.psrav.w.256" => "__builtin_ia32_psrav16hi_mask",
+        "llvm.x86.avx512.psrav.w.128" => "__builtin_ia32_psrav8hi_mask",
+        "llvm.x86.avx512.vpermi2var.hi.512" => "__builtin_ia32_vpermt2varhi512_mask",
+        "llvm.x86.avx512.vpermi2var.hi.256" => "__builtin_ia32_vpermt2varhi256_mask",
+        "llvm.x86.avx512.vpermi2var.hi.128" => "__builtin_ia32_vpermt2varhi128_mask",
+        "llvm.x86.avx512.permvar.hi.512" => "__builtin_ia32_permvarhi512_mask",
+        "llvm.x86.avx512.permvar.hi.256" => "__builtin_ia32_permvarhi256_mask",
+        "llvm.x86.avx512.permvar.hi.128" => "__builtin_ia32_permvarhi128_mask",
+        "llvm.x86.avx512.pshuf.b.512" => "__builtin_ia32_pshufb512_mask",
+        "llvm.x86.avx512.dbpsadbw.512" => "__builtin_ia32_dbpsadbw512_mask",
+        "llvm.x86.avx512.dbpsadbw.256" => "__builtin_ia32_dbpsadbw256_mask",
+        "llvm.x86.avx512.dbpsadbw.128" => "__builtin_ia32_dbpsadbw128_mask",
+        "llvm.x86.avx512.vpmadd52h.uq.512" => "__builtin_ia32_vpmadd52huq512_mask",
+        "llvm.x86.avx512.vpmadd52l.uq.512" => "__builtin_ia32_vpmadd52luq512_mask",
+        "llvm.x86.avx512.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256_mask",
+        "llvm.x86.avx512.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256_mask",
+        "llvm.x86.avx512.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128_mask",
+        "llvm.x86.avx512.vpdpwssd.512" => "__builtin_ia32_vpdpwssd_v16si",
+        "llvm.x86.avx512.vpdpwssd.256" => "__builtin_ia32_vpdpwssd_v8si",
+        "llvm.x86.avx512.vpdpwssd.128" => "__builtin_ia32_vpdpwssd_v4si",
+        "llvm.x86.avx512.vpdpwssds.512" => "__builtin_ia32_vpdpwssds_v16si",
+        "llvm.x86.avx512.vpdpwssds.256" => "__builtin_ia32_vpdpwssds_v8si",
+        "llvm.x86.avx512.vpdpwssds.128" => "__builtin_ia32_vpdpwssds_v4si",
+        "llvm.x86.avx512.vpdpbusd.512" => "__builtin_ia32_vpdpbusd_v16si",
+        "llvm.x86.avx512.vpdpbusd.256" => "__builtin_ia32_vpdpbusd_v8si",
+        "llvm.x86.avx512.vpdpbusd.128" => "__builtin_ia32_vpdpbusd_v4si",
+        "llvm.x86.avx512.vpdpbusds.512" => "__builtin_ia32_vpdpbusds_v16si",
+        "llvm.x86.avx512.vpdpbusds.256" => "__builtin_ia32_vpdpbusds_v8si",
+        "llvm.x86.avx512.vpdpbusds.128" => "__builtin_ia32_vpdpbusds_v4si",
+
         // NOTE: this file is generated by https://github.com/GuillaumeGomez/llvmint/blob/master/generate_list.py
         _ => include!("archs.rs"),
     };
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
index 49be6c649..94dc8c9e9 100644
--- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
@@ -1,6 +1,9 @@
 pub mod llvm;
 mod simd;
 
+#[cfg(feature="master")]
+use std::iter;
+
 use gccjit::{ComparisonOp, Function, RValue, ToRValue, Type, UnaryOp, FunctionType};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::base::wants_msvc_seh;
@@ -8,15 +11,23 @@ use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::{ArgAbiMethods, BaseTypeMethods, BuilderMethods, ConstMethods, IntrinsicCallMethods};
+#[cfg(feature="master")]
+use rustc_codegen_ssa::traits::{DerivedTypeMethods, MiscMethods};
 use rustc_middle::bug;
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_middle::ty::layout::LayoutOf;
+#[cfg(feature="master")]
+use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
 use rustc_span::{Span, Symbol, symbol::kw, sym};
 use rustc_target::abi::HasDataLayout;
 use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
 use rustc_target::spec::PanicStrategy;
+#[cfg(feature="master")]
+use rustc_target::spec::abi::Abi;
 
 use crate::abi::GccType;
+#[cfg(feature="master")]
+use crate::abi::FnAbiGccExt;
 use crate::builder::Builder;
 use crate::common::{SignType, TypeReflection};
 use crate::context::CodegenCx;
@@ -68,6 +79,8 @@ fn get_simple_intrinsic<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, name: Symbol) ->
         sym::nearbyintf64 => "nearbyint",
         sym::roundf32 => "roundf",
         sym::roundf64 => "round",
+        sym::roundevenf32 => "roundevenf",
+        sym::roundevenf64 => "roundeven",
         sym::abort => "abort",
         _ => return None,
     };
@@ -91,7 +104,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         let name = tcx.item_name(def_id);
         let name_str = name.as_str();
 
-        let llret_ty = self.layout_of(ret_ty).gcc_type(self, true);
+        let llret_ty = self.layout_of(ret_ty).gcc_type(self);
         let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
 
         let simple = get_simple_intrinsic(self, name);
@@ -404,7 +417,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
     /// Gets the LLVM type for a place of the original Rust type of
     /// this argument/return, i.e., the result of `type_of::type_of`.
     fn memory_ty(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc> {
-        self.layout.gcc_type(cx, true)
+        self.layout.gcc_type(cx)
     }
 
     /// Stores a direct/indirect value described by this ArgAbi into a
@@ -1120,10 +1133,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     }
 }
 
-fn try_intrinsic<'gcc, 'tcx>(bx: &mut Builder<'_, 'gcc, 'tcx>, try_func: RValue<'gcc>, data: RValue<'gcc>, _catch_func: RValue<'gcc>, dest: RValue<'gcc>) {
-    // NOTE: the `|| true` here is to use the panic=abort strategy with panic=unwind too
-    if bx.sess().panic_strategy() == PanicStrategy::Abort || true {
-        // TODO(bjorn3): Properly implement unwinding and remove the `|| true` once this is done.
+fn try_intrinsic<'a, 'b, 'gcc, 'tcx>(bx: &'b mut Builder<'a, 'gcc, 'tcx>, try_func: RValue<'gcc>, data: RValue<'gcc>, _catch_func: RValue<'gcc>, dest: RValue<'gcc>) {
+    if bx.sess().panic_strategy() == PanicStrategy::Abort {
         bx.call(bx.type_void(), None, try_func, &[data], None);
         // Return 0 unconditionally from the intrinsic call;
         // we can never unwind.
@@ -1134,6 +1145,141 @@ fn try_intrinsic<'gcc, 'tcx>(bx: &mut Builder<'_, 'gcc, 'tcx>, try_func: RValue<
         unimplemented!();
     }
     else {
+        #[cfg(feature="master")]
+        codegen_gnu_try(bx, try_func, data, _catch_func, dest);
+        #[cfg(not(feature="master"))]
         unimplemented!();
     }
 }
+
+// Definition of the standard `try` function for Rust using the GNU-like model
+// of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke`
+// instructions).
+//
+// This codegen is a little surprising because we always call a shim
+// function instead of inlining the call to `invoke` manually here. This is done
+// because in LLVM we're only allowed to have one personality per function
+// definition. The call to the `try` intrinsic is being inlined into the
+// function calling it, and that function may already have other personality
+// functions in play. By calling a shim we're guaranteed that our shim will have
+// the right personality function.
+#[cfg(feature="master")]
+fn codegen_gnu_try<'gcc>(bx: &mut Builder<'_, 'gcc, '_>, try_func: RValue<'gcc>, data: RValue<'gcc>, catch_func: RValue<'gcc>, dest: RValue<'gcc>) {
+    let cx: &CodegenCx<'gcc, '_> = bx.cx;
+    let (llty, func) = get_rust_try_fn(cx, &mut |mut bx| {
+        // Codegens the shims described above:
+        //
+        //   bx:
+        //      invoke %try_func(%data) normal %normal unwind %catch
+        //
+        //   normal:
+        //      ret 0
+        //
+        //   catch:
+        //      (%ptr, _) = landingpad
+        //      call %catch_func(%data, %ptr)
+        //      ret 1
+        let then = bx.append_sibling_block("then");
+        let catch = bx.append_sibling_block("catch");
+
+        let func = bx.current_func();
+        let try_func = func.get_param(0).to_rvalue();
+        let data = func.get_param(1).to_rvalue();
+        let catch_func = func.get_param(2).to_rvalue();
+        let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void());
+
+        let current_block = bx.block.clone();
+
+        bx.switch_to_block(then);
+        bx.ret(bx.const_i32(0));
+
+        // Type indicator for the exception being thrown.
+        //
+        // The value is a pointer to the exception object
+        // being thrown.
+        bx.switch_to_block(catch);
+        bx.set_personality_fn(bx.eh_personality());
+
+        let eh_pointer_builtin = bx.cx.context.get_target_builtin_function("__builtin_eh_pointer");
+        let zero = bx.cx.context.new_rvalue_zero(bx.int_type);
+        let ptr = bx.cx.context.new_call(None, eh_pointer_builtin, &[zero]);
+        let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void());
+        bx.call(catch_ty, None, catch_func, &[data, ptr], None);
+        bx.ret(bx.const_i32(1));
+
+        // NOTE: the blocks must be filled before adding the try/catch, otherwise gcc will not
+        // generate a try/catch.
+        // FIXME(antoyo): add a check in the libgccjit API to prevent this.
+        bx.switch_to_block(current_block);
+        bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None);
+    });
+
+    let func = unsafe { std::mem::transmute(func) };
+
+    // Note that no invoke is used here because by definition this function
+    // can't panic (that's what it's catching).
+    let ret = bx.call(llty, None, func, &[try_func, data, catch_func], None);
+    let i32_align = bx.tcx().data_layout.i32_align.abi;
+    bx.store(ret, dest, i32_align);
+}
+
+
+// Helper function used to get a handle to the `__rust_try` function used to
+// catch exceptions.
+//
+// This function is only generated once and is then cached.
+#[cfg(feature="master")]
+fn get_rust_try_fn<'a, 'gcc, 'tcx>(cx: &'a CodegenCx<'gcc, 'tcx>, codegen: &mut dyn FnMut(Builder<'a, 'gcc, 'tcx>)) -> (Type<'gcc>, Function<'gcc>) {
+    if let Some(llfn) = cx.rust_try_fn.get() {
+        return llfn;
+    }
+
+    // Define the type up front for the signature of the rust_try function.
+    let tcx = cx.tcx;
+    let i8p = tcx.mk_mut_ptr(tcx.types.i8);
+    // `unsafe fn(*mut i8) -> ()`
+    let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig(
+        iter::once(i8p),
+        tcx.mk_unit(),
+        false,
+        rustc_hir::Unsafety::Unsafe,
+        Abi::Rust,
+    )));
+    // `unsafe fn(*mut i8, *mut i8) -> ()`
+    let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig(
+        [i8p, i8p].iter().cloned(),
+        tcx.mk_unit(),
+        false,
+        rustc_hir::Unsafety::Unsafe,
+        Abi::Rust,
+    )));
+    // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32`
+    let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig(
+        [try_fn_ty, i8p, catch_fn_ty],
+        tcx.types.i32,
+        false,
+        rustc_hir::Unsafety::Unsafe,
+        Abi::Rust,
+    ));
+    let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
+    cx.rust_try_fn.set(Some(rust_try));
+    rust_try
+}
+
+// Helper function to give a Block to a closure to codegen a shim function.
+// This is currently primarily used for the `try` intrinsic functions above.
+#[cfg(feature="master")]
+fn gen_fn<'a, 'gcc, 'tcx>(cx: &'a CodegenCx<'gcc, 'tcx>, name: &str, rust_fn_sig: ty::PolyFnSig<'tcx>, codegen: &mut dyn FnMut(Builder<'a, 'gcc, 'tcx>)) -> (Type<'gcc>, Function<'gcc>) {
+    let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
+    let (typ, _, _, _) = fn_abi.gcc_type(cx);
+    // FIXME(eddyb) find a nicer way to do this.
+    cx.linkage.set(FunctionType::Internal);
+    let func = cx.declare_fn(name, fn_abi);
+    let func_val = unsafe { std::mem::transmute(func) };
+    cx.set_frame_pointer_type(func_val);
+    cx.apply_target_cpu_attr(func_val);
+    let block = Builder::append_block(cx, func_val, "entry-block");
+    let bx = Builder::build(cx, block);
+    codegen(bx);
+    (typ, func)
+}
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
index cb8168b40..b59c3a64f 100644
--- a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
@@ -1,8 +1,13 @@
-use std::cmp::Ordering;
+#[cfg(feature="master")]
+use gccjit::{ComparisonOp, UnaryOp};
+use gccjit::ToRValue;
+use gccjit::{BinaryOp, RValue, Type};
 
-use gccjit::{BinaryOp, RValue, ToRValue, Type};
 use rustc_codegen_ssa::base::compare_simd_types;
-use rustc_codegen_ssa::common::TypeKind;
+use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
+#[cfg(feature="master")]
+use rustc_codegen_ssa::errors::ExpectedPointerMutability;
+use rustc_codegen_ssa::errors::InvalidMonomorphization;
 use rustc_codegen_ssa::mir::operand::OperandRef;
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods};
@@ -14,18 +19,21 @@ use rustc_span::{sym, Span, Symbol};
 use rustc_target::abi::Align;
 
 use crate::builder::Builder;
+#[cfg(feature="master")]
+use crate::context::CodegenCx;
+#[cfg(feature="master")]
+use crate::errors::{InvalidMonomorphizationExpectedSignedUnsigned, InvalidMonomorphizationInsertedType};
 use crate::errors::{
-    InvalidMonomorphizationExpectedSignedUnsigned, InvalidMonomorphizationExpectedSimd,
-    InvalidMonomorphizationInsertedType, InvalidMonomorphizationInvalidBitmask,
+    InvalidMonomorphizationExpectedSimd,
+    InvalidMonomorphizationInvalidBitmask,
     InvalidMonomorphizationInvalidFloatVector, InvalidMonomorphizationMaskType,
     InvalidMonomorphizationMismatchedLengths, InvalidMonomorphizationNotFloat,
     InvalidMonomorphizationReturnElement, InvalidMonomorphizationReturnIntegerType,
     InvalidMonomorphizationReturnLength, InvalidMonomorphizationReturnLengthInputType,
     InvalidMonomorphizationReturnType, InvalidMonomorphizationSimdShuffle,
-    InvalidMonomorphizationUnrecognized, InvalidMonomorphizationUnsupportedCast,
-    InvalidMonomorphizationUnsupportedElement, InvalidMonomorphizationUnsupportedOperation,
+    InvalidMonomorphizationUnrecognized, InvalidMonomorphizationUnsupportedElement,
+    InvalidMonomorphizationUnsupportedOperation,
 };
-use crate::intrinsic;
 
 pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
     bx: &mut Builder<'a, 'gcc, 'tcx>,
@@ -105,14 +113,19 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         let arg1_vector_type = arg1_type.unqualified().dyncast_vector().expect("vector type");
         let arg1_element_type = arg1_vector_type.get_element_type();
 
+        // NOTE: since the arguments can be vectors of floats, make sure the mask is a vector of
+        // integer.
+        let mask_element_type = bx.type_ix(arg1_element_type.get_size() as u64 * 8);
+        let vector_mask_type = bx.context.new_vector_type(mask_element_type, arg1_vector_type.get_num_units() as u64);
+
         let mut elements = vec![];
         let one = bx.context.new_rvalue_one(mask.get_type());
         for _ in 0..len {
-            let element = bx.context.new_cast(None, mask & one, arg1_element_type);
+            let element = bx.context.new_cast(None, mask & one, mask_element_type);
             elements.push(element);
             mask = mask >> one;
         }
-        let vector_mask = bx.context.new_rvalue_from_vector(None, arg1_type, &elements);
+        let vector_mask = bx.context.new_rvalue_from_vector(None, vector_mask_type, &elements);
 
         return Ok(bx.vector_select(vector_mask, arg1, args[2].immediate()));
     }
@@ -210,48 +223,12 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         let vector = args[0].immediate();
         let index = args[1].immediate();
         let value = args[2].immediate();
-        // TODO(antoyo): use a recursive unqualified() here.
-        let vector_type = vector.get_type().unqualified().dyncast_vector().expect("vector type");
-        let element_type = vector_type.get_element_type();
-        // NOTE: we cannot cast to an array and assign to its element here because the value might
-        // not be an l-value. So, call a builtin to set the element.
-        // TODO(antoyo): perhaps we could create a new vector or maybe there's a GIMPLE instruction for that?
-        // TODO(antoyo): don't use target specific builtins here.
-        let func_name = match in_len {
-            2 => {
-                if element_type == bx.i64_type {
-                    "__builtin_ia32_vec_set_v2di"
-                } else {
-                    unimplemented!();
-                }
-            }
-            4 => {
-                if element_type == bx.i32_type {
-                    "__builtin_ia32_vec_set_v4si"
-                } else {
-                    unimplemented!();
-                }
-            }
-            8 => {
-                if element_type == bx.i16_type {
-                    "__builtin_ia32_vec_set_v8hi"
-                } else {
-                    unimplemented!();
-                }
-            }
-            _ => unimplemented!("Len: {}", in_len),
-        };
-        let builtin = bx.context.get_target_builtin_function(func_name);
-        let param1_type = builtin.get_param(0).to_rvalue().get_type();
-        // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
-        let vector = bx.cx.bitcast_if_needed(vector, param1_type);
-        let result = bx.context.new_call(
-            None,
-            builtin,
-            &[vector, value, bx.context.new_cast(None, index, bx.int_type)],
-        );
-        // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
-        return Ok(bx.context.new_bitcast(None, result, vector.get_type()));
+        let variable = bx.current_func().new_local(None, vector.get_type(), "new_vector");
+        bx.llbb().add_assignment(None, variable, vector);
+        let lvalue = bx.context.new_vector_access(None, variable.to_rvalue(), index);
+        // TODO(antoyo): if simd_insert is constant, use BIT_REF.
+        bx.llbb().add_assignment(None, lvalue, value);
+        return Ok(variable.to_rvalue());
     }
 
     #[cfg(feature = "master")]
@@ -280,7 +257,8 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         return Ok(bx.vector_select(args[0].immediate(), args[1].immediate(), args[2].immediate()));
     }
 
-    if name == sym::simd_cast {
+    #[cfg(feature="master")]
+    if name == sym::simd_cast || name == sym::simd_as {
         require_simd!(ret_ty, "return");
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
         require!(
@@ -301,125 +279,40 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
         enum Style {
             Float,
-            Int(/* is signed? */ bool),
+            Int,
             Unsupported,
         }
 
-        let (in_style, in_width) = match in_elem.kind() {
-            // vectors of pointer-sized integers should've been
-            // disallowed before here, so this unwrap is safe.
-            ty::Int(i) => (
-                Style::Int(true),
-                i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
-            ),
-            ty::Uint(u) => (
-                Style::Int(false),
-                u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
-            ),
-            ty::Float(f) => (Style::Float, f.bit_width()),
-            _ => (Style::Unsupported, 0),
-        };
-        let (out_style, out_width) = match out_elem.kind() {
-            ty::Int(i) => (
-                Style::Int(true),
-                i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
-            ),
-            ty::Uint(u) => (
-                Style::Int(false),
-                u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
-            ),
-            ty::Float(f) => (Style::Float, f.bit_width()),
-            _ => (Style::Unsupported, 0),
-        };
-
-        let extend = |in_type, out_type| {
-            let vector_type = bx.context.new_vector_type(out_type, 8);
-            let vector = args[0].immediate();
-            let array_type = bx.context.new_array_type(None, in_type, 8);
-            // TODO(antoyo): switch to using new_vector_access or __builtin_convertvector for vector casting.
-            let array = bx.context.new_bitcast(None, vector, array_type);
-
-            let cast_vec_element = |index| {
-                let index = bx.context.new_rvalue_from_int(bx.int_type, index);
-                bx.context.new_cast(
-                    None,
-                    bx.context.new_array_access(None, array, index).to_rvalue(),
-                    out_type,
-                )
+        let in_style =
+            match in_elem.kind() {
+                ty::Int(_) | ty::Uint(_) => Style::Int,
+                ty::Float(_) => Style::Float,
+                 _ => Style::Unsupported,
             };
 
-            bx.context.new_rvalue_from_vector(
-                None,
-                vector_type,
-                &[
-                    cast_vec_element(0),
-                    cast_vec_element(1),
-                    cast_vec_element(2),
-                    cast_vec_element(3),
-                    cast_vec_element(4),
-                    cast_vec_element(5),
-                    cast_vec_element(6),
-                    cast_vec_element(7),
-                ],
-            )
-        };
+        let out_style =
+            match out_elem.kind() {
+                ty::Int(_) | ty::Uint(_) => Style::Int,
+                ty::Float(_) => Style::Float,
+                 _ => Style::Unsupported,
+            };
 
         match (in_style, out_style) {
-            (Style::Int(in_is_signed), Style::Int(_)) => {
-                return Ok(match in_width.cmp(&out_width) {
-                    Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
-                    Ordering::Equal => args[0].immediate(),
-                    Ordering::Less => {
-                        if in_is_signed {
-                            match (in_width, out_width) {
-                                // FIXME(antoyo): the function _mm_cvtepi8_epi16 should directly
-                                // call an intrinsic equivalent to __builtin_ia32_pmovsxbw128 so that
-                                // we can generate a call to it.
-                                (8, 16) => extend(bx.i8_type, bx.i16_type),
-                                (8, 32) => extend(bx.i8_type, bx.i32_type),
-                                (8, 64) => extend(bx.i8_type, bx.i64_type),
-                                (16, 32) => extend(bx.i16_type, bx.i32_type),
-                                (32, 64) => extend(bx.i32_type, bx.i64_type),
-                                (16, 64) => extend(bx.i16_type, bx.i64_type),
-                                _ => unimplemented!("in: {}, out: {}", in_width, out_width),
-                            }
-                        } else {
-                            match (in_width, out_width) {
-                                (8, 16) => extend(bx.u8_type, bx.u16_type),
-                                (8, 32) => extend(bx.u8_type, bx.u32_type),
-                                (8, 64) => extend(bx.u8_type, bx.u64_type),
-                                (16, 32) => extend(bx.u16_type, bx.u32_type),
-                                (16, 64) => extend(bx.u16_type, bx.u64_type),
-                                (32, 64) => extend(bx.u32_type, bx.u64_type),
-                                _ => unimplemented!("in: {}, out: {}", in_width, out_width),
-                            }
-                        }
+            (Style::Unsupported, Style::Unsupported) => {
+                require!(
+                    false,
+                    InvalidMonomorphization::UnsupportedCast {
+                        span,
+                        name,
+                        in_ty,
+                        in_elem,
+                        ret_ty,
+                        out_elem
                     }
-                });
-            }
-            (Style::Int(_), Style::Float) => {
-                // TODO: add support for internal functions in libgccjit to get access to IFN_VEC_CONVERT which is
-                // doing like __builtin_convertvector?
-                // Or maybe provide convert_vector as an API since it might not easy to get the
-                // types of internal functions.
-                unimplemented!();
-            }
-            (Style::Float, Style::Int(_)) => {
-                unimplemented!();
-            }
-            (Style::Float, Style::Float) => {
-                unimplemented!();
-            }
-            _ => { /* Unsupported. Fallthrough. */ }
+                );
+            },
+            _ => return Ok(bx.context.convert_vector(None, args[0].immediate(), llret_ty)),
         }
-        return_error!(InvalidMonomorphizationUnsupportedCast {
-            span,
-            name,
-            in_ty,
-            in_elem,
-            ret_ty,
-            out_elem
-        });
     }
 
     macro_rules! arith_binary {
@@ -436,6 +329,71 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         }
     }
 
+    if name == sym::simd_bitmask {
+        // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a
+        // vector mask and returns the most significant bit (MSB) of each lane in the form
+        // of either:
+        // * an unsigned integer
+        // * an array of `u8`
+        // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
+        //
+        // The bit order of the result depends on the byte endianness, LSB-first for little
+        // endian and MSB-first for big endian.
+
+        let vector = args[0].immediate();
+        let vector_type = vector.get_type().dyncast_vector().expect("vector type");
+        let elem_type = vector_type.get_element_type();
+
+        let expected_int_bits = in_len.max(8);
+        let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64);
+
+        // FIXME(antoyo): that's not going to work for masks bigger than 128 bits.
+        let result_type = bx.type_ix(expected_int_bits);
+        let mut result = bx.context.new_rvalue_zero(result_type);
+
+        let elem_size = elem_type.get_size() * 8;
+        let sign_shift = bx.context.new_rvalue_from_int(elem_type, elem_size as i32 - 1);
+        let one = bx.context.new_rvalue_one(elem_type);
+
+        let mut shift = 0;
+        for i in 0..in_len {
+            let elem = bx.extract_element(vector, bx.context.new_rvalue_from_int(bx.int_type, i as i32));
+            let shifted = elem >> sign_shift;
+            let masked = shifted & one;
+            result = result | (bx.context.new_cast(None, masked, result_type) << bx.context.new_rvalue_from_int(result_type, shift));
+            shift += 1;
+        }
+
+        match ret_ty.kind() {
+            ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
+                // Zero-extend iN to the bitmask type:
+                return Ok(result);
+            }
+            ty::Array(elem, len)
+                if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
+                    && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all())
+                        == Some(expected_bytes) =>
+            {
+                // Zero-extend iN to the array length:
+                let ze = bx.zext(result, bx.type_ix(expected_bytes * 8));
+
+                // Convert the integer to a byte array
+                let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE);
+                bx.store(ze, ptr, Align::ONE);
+                let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
+                let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty));
+                return Ok(bx.load(array_ty, ptr, Align::ONE));
+            }
+            _ => return_error!(InvalidMonomorphization::CannotReturn {
+                span,
+                name,
+                ret_ty,
+                expected_int_bits,
+                expected_bytes
+            }),
+        }
+    }
+
     fn simd_simple_float_intrinsic<'gcc, 'tcx>(
         name: Symbol,
         in_elem: Ty<'_>,
@@ -451,55 +409,66 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 return Err(());
             }};
         }
-        let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() {
-            let elem_ty = bx.cx.type_float_from_ty(*f);
-            match f.bit_width() {
-                32 => ("f32", elem_ty),
-                64 => ("f64", elem_ty),
-                _ => {
-                    return_error!(InvalidMonomorphizationInvalidFloatVector {
-                        span,
-                        name,
-                        elem_ty: f.name_str(),
-                        vec_ty: in_ty
-                    });
+        let (elem_ty_str, elem_ty) =
+            if let ty::Float(f) = in_elem.kind() {
+                let elem_ty = bx.cx.type_float_from_ty(*f);
+                match f.bit_width() {
+                    32 => ("f", elem_ty),
+                    64 => ("", elem_ty),
+                    _ => {
+                        return_error!(InvalidMonomorphizationInvalidFloatVector { span, name, elem_ty: f.name_str(), vec_ty: in_ty });
+                    }
                 }
             }
-        } else {
-            return_error!(InvalidMonomorphizationNotFloat { span, name, ty: in_ty });
-        };
+            else {
+                return_error!(InvalidMonomorphizationNotFloat { span, name, ty: in_ty });
+            };
 
         let vec_ty = bx.cx.type_vector(elem_ty, in_len);
 
-        let (intr_name, fn_ty) = match name {
-            sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), // TODO(antoyo): pand with 170141183420855150465331762880109871103
-            sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
-            sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
-            sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
-            sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
-            _ => return_error!(InvalidMonomorphizationUnrecognized { span, name }),
-        };
-        let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
-        let function = intrinsic::llvm::intrinsic(llvm_name, &bx.cx);
-        let function: RValue<'gcc> = unsafe { std::mem::transmute(function) };
-        let c = bx.call(
-            fn_ty,
-            None,
-            function,
-            &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
-            None,
-        );
+        let intr_name =
+            match name {
+                sym::simd_ceil => "ceil",
+                sym::simd_fabs => "fabs", // TODO(antoyo): pand with 170141183420855150465331762880109871103
+                sym::simd_fcos => "cos",
+                sym::simd_fexp2 => "exp2",
+                sym::simd_fexp => "exp",
+                sym::simd_flog10 => "log10",
+                sym::simd_flog2 => "log2",
+                sym::simd_flog => "log",
+                sym::simd_floor => "floor",
+                sym::simd_fma => "fma",
+                sym::simd_fpowi => "__builtin_powi",
+                sym::simd_fpow => "pow",
+                sym::simd_fsin => "sin",
+                sym::simd_fsqrt => "sqrt",
+                sym::simd_round => "round",
+                sym::simd_trunc => "trunc",
+                _ => return_error!(InvalidMonomorphizationUnrecognized { span, name })
+            };
+        let builtin_name = format!("{}{}", intr_name, elem_ty_str);
+        let funcs = bx.cx.functions.borrow();
+        let function = funcs.get(&builtin_name).unwrap_or_else(|| panic!("unable to find builtin function {}", builtin_name));
+
+        // TODO(antoyo): add platform-specific behavior here for architectures that have these
+        // intrinsics as instructions (for instance, gpus)
+        let mut vector_elements = vec![];
+        for i in 0..in_len {
+            let index = bx.context.new_rvalue_from_long(bx.ulong_type, i as i64);
+            // we have to treat fpowi specially, since fpowi's second argument is always an i32
+            let arguments = if name == sym::simd_fpowi {
+                vec![
+                    bx.extract_element(args[0].immediate(), index).to_rvalue(),
+                    args[1].immediate(),
+                ]
+            } else {
+                args.iter()
+                    .map(|arg| bx.extract_element(arg.immediate(), index).to_rvalue())
+                    .collect()
+            };
+            vector_elements.push(bx.context.new_call(None, *function, &arguments));
+        }
+        let c = bx.context.new_rvalue_from_vector(None, vec_ty, &vector_elements);
         Ok(c)
     }
 
@@ -525,6 +494,297 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
     }
 
+    #[cfg(feature="master")]
+    fn vector_ty<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, elem_ty: Ty<'tcx>, vec_len: u64) -> Type<'gcc> {
+        // FIXME: use cx.layout_of(ty).llvm_type() ?
+        let elem_ty = match *elem_ty.kind() {
+            ty::Int(v) => cx.type_int_from_ty(v),
+            ty::Uint(v) => cx.type_uint_from_ty(v),
+            ty::Float(v) => cx.type_float_from_ty(v),
+            _ => unreachable!(),
+        };
+        cx.type_vector(elem_ty, vec_len)
+    }
+
+    #[cfg(feature="master")]
+    fn gather<'a, 'gcc, 'tcx>(default: RValue<'gcc>, pointers: RValue<'gcc>, mask: RValue<'gcc>, pointer_count: usize, bx: &mut Builder<'a, 'gcc, 'tcx>, in_len: u64, underlying_ty: Ty<'tcx>, invert: bool) -> RValue<'gcc> {
+        let vector_type =
+            if pointer_count > 1 {
+                bx.context.new_vector_type(bx.usize_type, in_len)
+            }
+            else {
+                vector_ty(bx, underlying_ty, in_len)
+            };
+        let elem_type = vector_type.dyncast_vector().expect("vector type").get_element_type();
+
+        let mut values = vec![];
+        for i in 0..in_len {
+            let index = bx.context.new_rvalue_from_long(bx.i32_type, i as i64);
+            let int = bx.context.new_vector_access(None, pointers, index).to_rvalue();
+
+            let ptr_type = elem_type.make_pointer();
+            let ptr = bx.context.new_bitcast(None, int, ptr_type);
+            let value = ptr.dereference(None).to_rvalue();
+            values.push(value);
+        }
+
+        let vector = bx.context.new_rvalue_from_vector(None, vector_type, &values);
+
+        let mut mask_types = vec![];
+        let mut mask_values = vec![];
+        for i in 0..in_len {
+            let index = bx.context.new_rvalue_from_long(bx.i32_type, i as i64);
+            mask_types.push(bx.context.new_field(None, bx.i32_type, "m"));
+            let mask_value = bx.context.new_vector_access(None, mask, index).to_rvalue();
+            let masked = bx.context.new_rvalue_from_int(bx.i32_type, in_len as i32) & mask_value;
+            let value = index + masked;
+            mask_values.push(value);
+        }
+        let mask_type = bx.context.new_struct_type(None, "mask_type", &mask_types);
+        let mask = bx.context.new_struct_constructor(None, mask_type.as_type(), None, &mask_values);
+
+        if invert {
+            bx.shuffle_vector(vector, default, mask)
+        }
+        else {
+            bx.shuffle_vector(default, vector, mask)
+        }
+    }
+
+    #[cfg(feature="master")]
+    if name == sym::simd_gather {
+        // simd_gather(values: <N x T>, pointers: <N x *_ T>,
+        //             mask: <N x i{M}>) -> <N x T>
+        // * N: number of elements in the input vectors
+        // * T: type of the element to load
+        // * M: any integer width is supported, will be truncated to i1
+
+        // All types must be simd vector types
+        require_simd!(in_ty, "first");
+        require_simd!(arg_tys[1], "second");
+        require_simd!(arg_tys[2], "third");
+        require_simd!(ret_ty, "return");
+
+        // Of the same length:
+        let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::SecondArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[1],
+                out_len
+            }
+        );
+        require!(
+            in_len == out_len2,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[2],
+                out_len: out_len2
+            }
+        );
+
+        // The return type must match the first argument type
+        require!(
+            ret_ty == in_ty,
+            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
+        );
+
+        // This counts how many pointers
+        fn ptr_count(t: Ty<'_>) -> usize {
+            match t.kind() {
+                ty::RawPtr(p) => 1 + ptr_count(p.ty),
+                _ => 0,
+            }
+        }
+
+        // Non-ptr type
+        fn non_ptr(t: Ty<'_>) -> Ty<'_> {
+            match t.kind() {
+                ty::RawPtr(p) => non_ptr(p.ty),
+                _ => t,
+            }
+        }
+
+        // The second argument must be a simd vector with an element type that's a pointer
+        // to the element type of the first argument
+        let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
+        let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (pointer_count, underlying_ty) = match element_ty1.kind() {
+            ty::RawPtr(p) if p.ty == in_elem => (ptr_count(element_ty1), non_ptr(element_ty1)),
+            _ => {
+                require!(
+                    false,
+                    InvalidMonomorphization::ExpectedElementType {
+                        span,
+                        name,
+                        expected_element: element_ty1,
+                        second_arg: arg_tys[1],
+                        in_elem,
+                        in_ty,
+                        mutability: ExpectedPointerMutability::Not,
+                    }
+                );
+                unreachable!();
+            }
+        };
+        assert!(pointer_count > 0);
+        assert_eq!(pointer_count - 1, ptr_count(element_ty0));
+        assert_eq!(underlying_ty, non_ptr(element_ty0));
+
+        // The element type of the third argument must be a signed integer type of any width:
+        let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
+        match element_ty2.kind() {
+            ty::Int(_) => (),
+            _ => {
+                require!(
+                    false,
+                    InvalidMonomorphization::ThirdArgElementType {
+                        span,
+                        name,
+                        expected_element: element_ty2,
+                        third_arg: arg_tys[2]
+                    }
+                );
+            }
+        }
+
+        return Ok(gather(args[0].immediate(), args[1].immediate(), args[2].immediate(), pointer_count, bx, in_len, underlying_ty, false));
+    }
+
+    #[cfg(feature="master")]
+    if name == sym::simd_scatter {
+        // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
+        //             mask: <N x i{M}>) -> ()
+        // * N: number of elements in the input vectors
+        // * T: type of the element to load
+        // * M: any integer width is supported, will be truncated to i1
+
+        // All types must be simd vector types
+        require_simd!(in_ty, "first");
+        require_simd!(arg_tys[1], "second");
+        require_simd!(arg_tys[2], "third");
+
+        // Of the same length:
+        let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
+        require!(
+            in_len == element_len1,
+            InvalidMonomorphization::SecondArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[1],
+                out_len: element_len1
+            }
+        );
+        require!(
+            in_len == element_len2,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[2],
+                out_len: element_len2
+            }
+        );
+
+        // This counts how many pointers
+        fn ptr_count(t: Ty<'_>) -> usize {
+            match t.kind() {
+                ty::RawPtr(p) => 1 + ptr_count(p.ty),
+                _ => 0,
+            }
+        }
+
+        // Non-ptr type
+        fn non_ptr(t: Ty<'_>) -> Ty<'_> {
+            match t.kind() {
+                ty::RawPtr(p) => non_ptr(p.ty),
+                _ => t,
+            }
+        }
+
+        // The second argument must be a simd vector with an element type that's a pointer
+        // to the element type of the first argument
+        let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
+        let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
+        let (pointer_count, underlying_ty) = match element_ty1.kind() {
+            ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => {
+                (ptr_count(element_ty1), non_ptr(element_ty1))
+            }
+            _ => {
+                require!(
+                    false,
+                    InvalidMonomorphization::ExpectedElementType {
+                        span,
+                        name,
+                        expected_element: element_ty1,
+                        second_arg: arg_tys[1],
+                        in_elem,
+                        in_ty,
+                        mutability: ExpectedPointerMutability::Mut,
+                    }
+                );
+                unreachable!();
+            }
+        };
+        assert!(pointer_count > 0);
+        assert_eq!(pointer_count - 1, ptr_count(element_ty0));
+        assert_eq!(underlying_ty, non_ptr(element_ty0));
+
+        // The element type of the third argument must be a signed integer type of any width:
+        match element_ty2.kind() {
+            ty::Int(_) => (),
+            _ => {
+                require!(
+                    false,
+                    InvalidMonomorphization::ThirdArgElementType {
+                        span,
+                        name,
+                        expected_element: element_ty2,
+                        third_arg: arg_tys[2]
+                    }
+                );
+            }
+        }
+
+        let result = gather(args[0].immediate(), args[1].immediate(), args[2].immediate(), pointer_count, bx, in_len, underlying_ty, true);
+
+        let pointers = args[1].immediate();
+
+        let vector_type =
+            if pointer_count > 1 {
+                bx.context.new_vector_type(bx.usize_type, in_len)
+            }
+            else {
+                vector_ty(bx, underlying_ty, in_len)
+            };
+        let elem_type = vector_type.dyncast_vector().expect("vector type").get_element_type();
+
+        for i in 0..in_len {
+            let index = bx.context.new_rvalue_from_int(bx.int_type, i as i32);
+            let value = bx.context.new_vector_access(None, result, index);
+
+            let int = bx.context.new_vector_access(None, pointers, index).to_rvalue();
+            let ptr_type = elem_type.make_pointer();
+            let ptr = bx.context.new_bitcast(None, int, ptr_type);
+            bx.llbb().add_assignment(None, ptr.dereference(None), value);
+        }
+
+        return Ok(bx.context.new_rvalue_zero(bx.i32_type));
+    }
+
     arith_binary! {
         simd_add: Uint, Int => add, Float => fadd;
         simd_sub: Uint, Int => sub, Float => fsub;
@@ -536,6 +796,8 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         simd_and: Uint, Int => and;
         simd_or: Uint, Int => or; // FIXME(antoyo): calling `or` might not work on vectors.
         simd_xor: Uint, Int => xor;
+        simd_fmin: Float => vector_fmin;
+        simd_fmax: Float => vector_fmax;
     }
 
     macro_rules! arith_unary {
@@ -562,10 +824,11 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         let rhs = args[1].immediate();
         let is_add = name == sym::simd_saturating_add;
         let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
-        let (signed, elem_width, elem_ty) = match *in_elem.kind() {
-            ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
-            ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
-            _ => {
+        let (signed, elem_width, elem_ty) =
+            match *in_elem.kind() {
+                ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits) / 8, bx.cx.type_int_from_ty(i)),
+                ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits) / 8, bx.cx.type_uint_from_ty(i)),
+                _ => {
                 return_error!(InvalidMonomorphizationExpectedSignedUnsigned {
                     span,
                     name,
@@ -574,33 +837,78 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 });
             }
         };
-        let builtin_name = match (signed, is_add, in_len, elem_width) {
-            (true, true, 32, 8) => "__builtin_ia32_paddsb256", // TODO(antoyo): cast arguments to unsigned.
-            (false, true, 32, 8) => "__builtin_ia32_paddusb256",
-            (true, true, 16, 16) => "__builtin_ia32_paddsw256",
-            (false, true, 16, 16) => "__builtin_ia32_paddusw256",
-            (true, false, 16, 16) => "__builtin_ia32_psubsw256",
-            (false, false, 16, 16) => "__builtin_ia32_psubusw256",
-            (true, false, 32, 8) => "__builtin_ia32_psubsb256",
-            (false, false, 32, 8) => "__builtin_ia32_psubusb256",
-            _ => unimplemented!(
-                "signed: {}, is_add: {}, in_len: {}, elem_width: {}",
-                signed,
-                is_add,
-                in_len,
-                elem_width
-            ),
-        };
-        let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
-
-        let func = bx.context.get_target_builtin_function(builtin_name);
-        let param1_type = func.get_param(0).to_rvalue().get_type();
-        let param2_type = func.get_param(1).to_rvalue().get_type();
-        let lhs = bx.cx.bitcast_if_needed(lhs, param1_type);
-        let rhs = bx.cx.bitcast_if_needed(rhs, param2_type);
-        let result = bx.context.new_call(None, func, &[lhs, rhs]);
-        // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
-        return Ok(bx.context.new_bitcast(None, result, vec_ty));
+
+        let result =
+            match (signed, is_add) {
+                (false, true) => {
+                    let res = lhs + rhs;
+                    let cmp = bx.context.new_comparison(None, ComparisonOp::LessThan, res, lhs);
+                    res | cmp
+                },
+                (true, true) => {
+                    // Algorithm from: https://codereview.stackexchange.com/questions/115869/saturated-signed-addition
+                    // TODO(antoyo): improve using conditional operators if possible.
+                    let arg_type = lhs.get_type();
+                    // TODO(antoyo): convert lhs and rhs to unsigned.
+                    let sum = lhs + rhs;
+                    let vector_type = arg_type.dyncast_vector().expect("vector type");
+                    let unit = vector_type.get_num_units();
+                    let a = bx.context.new_rvalue_from_int(elem_ty, ((elem_width as i32) << 3) - 1);
+                    let width = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![a; unit]);
+
+                    let xor1 = lhs ^ rhs;
+                    let xor2 = lhs ^ sum;
+                    let and = bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, xor1) & xor2;
+                    let mask = and >> width;
+
+                    let one = bx.context.new_rvalue_one(elem_ty);
+                    let ones = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![one; unit]);
+                    let shift1 = ones << width;
+                    let shift2 = sum >> width;
+                    let mask_min = shift1 ^ shift2;
+
+                    let and1 = bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, mask) & sum;
+                    let and2 = mask & mask_min;
+
+                    and1 + and2
+                },
+                (false, false) => {
+                    let res = lhs - rhs;
+                    let cmp = bx.context.new_comparison(None, ComparisonOp::LessThanEquals, res, lhs);
+                    res & cmp
+                },
+                (true, false) => {
+                    let arg_type = lhs.get_type();
+                    // TODO(antoyo): this uses the same algorithm from saturating add, but add the
+                    // negative of the right operand. Find a proper subtraction algorithm.
+                    let rhs = bx.context.new_unary_op(None, UnaryOp::Minus, arg_type, rhs);
+
+                    // TODO(antoyo): convert lhs and rhs to unsigned.
+                    let sum = lhs + rhs;
+                    let vector_type = arg_type.dyncast_vector().expect("vector type");
+                    let unit = vector_type.get_num_units();
+                    let a = bx.context.new_rvalue_from_int(elem_ty, ((elem_width as i32) << 3) - 1);
+                    let width = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![a; unit]);
+
+                    let xor1 = lhs ^ rhs;
+                    let xor2 = lhs ^ sum;
+                    let and = bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, xor1) & xor2;
+                    let mask = and >> width;
+
+                    let one = bx.context.new_rvalue_one(elem_ty);
+                    let ones = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![one; unit]);
+                    let shift1 = ones << width;
+                    let shift2 = sum >> width;
+                    let mask_min = shift1 ^ shift2;
+
+                    let and1 = bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, mask) & sum;
+                    let and2 = mask & mask_min;
+
+                    and1 + and2
+                }
+            };
+
+        return Ok(result);
     }
 
     macro_rules! arith_red {
@@ -650,33 +958,50 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         add,
         0.0 // TODO: Use this argument.
     );
-    arith_red!(simd_reduce_mul_unordered: BinaryOp::Mult, vector_reduce_fmul_fast, false, mul, 1.0);
+    arith_red!(
+        simd_reduce_mul_unordered: BinaryOp::Mult,
+        vector_reduce_fmul_fast,
+        false,
+        mul,
+        1.0
+    );
+    arith_red!(
+        simd_reduce_add_ordered: BinaryOp::Plus,
+        vector_reduce_fadd,
+        true,
+        add,
+        0.0
+    );
+    arith_red!(
+        simd_reduce_mul_ordered: BinaryOp::Mult,
+        vector_reduce_fmul,
+        true,
+        mul,
+        1.0
+    );
+
 
     macro_rules! minmax_red {
-        ($name:ident: $reduction:ident) => {
+        ($name:ident: $int_red:ident, $float_red:ident) => {
             if name == sym::$name {
                 require!(
                     ret_ty == in_elem,
                     InvalidMonomorphizationReturnType { span, name, in_elem, in_ty, ret_ty }
                 );
                 return match in_elem.kind() {
-                    ty::Int(_) | ty::Uint(_) | ty::Float(_) => {
-                        Ok(bx.$reduction(args[0].immediate()))
-                    }
-                    _ => return_error!(InvalidMonomorphizationUnsupportedElement {
-                        span,
-                        name,
-                        in_ty,
-                        elem_ty: in_elem,
-                        ret_ty
-                    }),
+                    ty::Int(_) | ty::Uint(_) => Ok(bx.$int_red(args[0].immediate())),
+                    ty::Float(_) => Ok(bx.$float_red(args[0].immediate())),
+                    _ => return_error!(InvalidMonomorphizationUnsupportedElement { span, name, in_ty, elem_ty: in_elem, ret_ty }),
                 };
             }
         };
     }
 
-    minmax_red!(simd_reduce_min: vector_reduce_min);
-    minmax_red!(simd_reduce_max: vector_reduce_max);
+    minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
+    minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
+    // TODO(sadlerap): revisit these intrinsics to generate more optimal reductions
+    minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin);
+    minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax);
 
     macro_rules! bitwise_red {
         ($name:ident : $op:expr, $boolean:expr) => {
@@ -699,15 +1024,12 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                         }),
                     }
 
-                    // boolean reductions operate on vectors of i1s:
-                    let i1 = bx.type_i1();
-                    let i1xn = bx.type_vector(i1, in_len as u64);
-                    bx.trunc(args[0].immediate(), i1xn)
+                    args[0].immediate()
                 };
                 return match in_elem.kind() {
                     ty::Int(_) | ty::Uint(_) => {
                         let r = bx.vector_reduce_op(input, $op);
-                        Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
+                        Ok(if !$boolean { r } else { bx.icmp(IntPredicate::IntNE, r, bx.context.new_rvalue_zero(r.get_type())) })
                     }
                     _ => return_error!(InvalidMonomorphizationUnsupportedElement {
                         span,
@@ -723,6 +1045,9 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
     bitwise_red!(simd_reduce_and: BinaryOp::BitwiseAnd, false);
     bitwise_red!(simd_reduce_or: BinaryOp::BitwiseOr, false);
+    bitwise_red!(simd_reduce_xor: BinaryOp::BitwiseXor, false);
+    bitwise_red!(simd_reduce_all: BinaryOp::BitwiseAnd, true);
+    bitwise_red!(simd_reduce_any: BinaryOp::BitwiseOr, true);
 
     unimplemented!("simd {}", name);
 }
diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs
index 44538b415..0b661505a 100644
--- a/compiler/rustc_codegen_gcc/src/lib.rs
+++ b/compiler/rustc_codegen_gcc/src/lib.rs
@@ -1,7 +1,7 @@
 /*
  * TODO(antoyo): implement equality in libgccjit based on https://zpz.github.io/blog/overloading-equality-operator-in-cpp-class-hierarchy/ (for type equality?)
  * TODO(antoyo): support #[inline] attributes.
- * TODO(antoyo): support LTO (gcc's equivalent to Thin LTO is enabled by -fwhopr: https://stackoverflow.com/questions/64954525/does-gcc-have-thin-lto).
+ * TODO(antoyo): support LTO (gcc's equivalent to Full LTO is -flto -flto-partition=one — https://documentation.suse.com/sbp/all/html/SBP-GCC-10/index.html).
  *
  * TODO(antoyo): remove the patches.
  */
@@ -23,6 +23,7 @@
 
 extern crate rustc_apfloat;
 extern crate rustc_ast;
+extern crate rustc_attr;
 extern crate rustc_codegen_ssa;
 extern crate rustc_data_structures;
 extern crate rustc_errors;
@@ -43,6 +44,7 @@ mod abi;
 mod allocator;
 mod archive;
 mod asm;
+mod attributes;
 mod back;
 mod base;
 mod builder;
@@ -85,7 +87,7 @@ use rustc_span::Symbol;
 use rustc_span::fatal_error::FatalError;
 use tempfile::TempDir;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub struct PrintOnPanic<F: Fn() -> String>(pub F);
 
@@ -314,9 +316,12 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
         .filter(|_feature| {
             // TODO(antoyo): implement a way to get enabled feature in libgccjit.
             // Probably using the equivalent of __builtin_cpu_supports.
+            // TODO(antoyo): maybe use whatever outputs the following command:
+            // gcc -march=native -Q --help=target
             #[cfg(feature="master")]
             {
-                _feature.contains("sse") || _feature.contains("avx")
+                // NOTE: the CPU in the CI doesn't support sse4a, so disable it to make the stdarch tests pass in the CI.
+                (_feature.contains("sse") || _feature.contains("avx")) && !_feature.contains("avx512") && !_feature.contains("sse4a")
             }
             #[cfg(not(feature="master"))]
             {
diff --git a/compiler/rustc_codegen_gcc/src/mono_item.rs b/compiler/rustc_codegen_gcc/src/mono_item.rs
index a7c868354..c1f634086 100644
--- a/compiler/rustc_codegen_gcc/src/mono_item.rs
+++ b/compiler/rustc_codegen_gcc/src/mono_item.rs
@@ -1,38 +1,66 @@
+#[cfg(feature="master")]
+use gccjit::{VarAttribute, FnAttribute};
 use rustc_codegen_ssa::traits::PreDefineMethods;
+use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::mono::{Linkage, Visibility};
 use rustc_middle::ty::{self, Instance, TypeVisitableExt};
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
-use rustc_span::def_id::DefId;
 
+use crate::attributes;
 use crate::base;
 use crate::context::CodegenCx;
 use crate::type_of::LayoutGccExt;
 
 impl<'gcc, 'tcx> PreDefineMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
-    fn predefine_static(&self, def_id: DefId, _linkage: Linkage, _visibility: Visibility, symbol_name: &str) {
+    #[cfg_attr(not(feature="master"), allow(unused_variables))]
+    fn predefine_static(&self, def_id: DefId, _linkage: Linkage, visibility: Visibility, symbol_name: &str) {
         let attrs = self.tcx.codegen_fn_attrs(def_id);
         let instance = Instance::mono(self.tcx, def_id);
         let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
-        let gcc_type = self.layout_of(ty).gcc_type(self, true);
+        let gcc_type = self.layout_of(ty).gcc_type(self);
 
         let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL);
         let global = self.define_global(symbol_name, gcc_type, is_tls, attrs.link_section);
+        #[cfg(feature="master")]
+        global.add_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility)));
 
-        // TODO(antoyo): set linkage and visibility.
+        // TODO(antoyo): set linkage.
         self.instances.borrow_mut().insert(instance, global);
     }
 
-    fn predefine_fn(&self, instance: Instance<'tcx>, linkage: Linkage, _visibility: Visibility, symbol_name: &str) {
+    #[cfg_attr(not(feature="master"), allow(unused_variables))]
+    fn predefine_fn(&self, instance: Instance<'tcx>, linkage: Linkage, visibility: Visibility, symbol_name: &str) {
         assert!(!instance.substs.needs_infer());
 
         let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
         self.linkage.set(base::linkage_to_gcc(linkage));
-        let _decl = self.declare_fn(symbol_name, &fn_abi);
+        let decl = self.declare_fn(symbol_name, &fn_abi);
         //let attrs = self.tcx.codegen_fn_attrs(instance.def_id());
 
+        attributes::from_fn_attrs(self, decl, instance);
+
+        // If we're compiling the compiler-builtins crate, e.g., the equivalent of
+        // compiler-rt, then we want to implicitly compile everything with hidden
+        // visibility as we're going to link this object all over the place but
+        // don't want the symbols to get exported.
+        if linkage != Linkage::Internal
+            && linkage != Linkage::Private
+            && self.tcx.is_compiler_builtins(LOCAL_CRATE)
+        {
+            #[cfg(feature="master")]
+            decl.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
+        }
+        else {
+            #[cfg(feature="master")]
+            decl.add_attribute(FnAttribute::Visibility(base::visibility_to_gcc(visibility)));
+        }
+
         // TODO(antoyo): call set_link_section() to allow initializing argc/argv.
         // TODO(antoyo): set unique comdat.
         // TODO(antoyo): use inline attribute from there in linkage.set() above.
+
+        self.functions.borrow_mut().insert(symbol_name.to_string(), decl);
+        self.function_instances.borrow_mut().insert(instance, unsafe { std::mem::transmute(decl) });
     }
 }
diff --git a/compiler/rustc_codegen_gcc/src/type_.rs b/compiler/rustc_codegen_gcc/src/type_.rs
index 89a415cdb..daa661f35 100644
--- a/compiler/rustc_codegen_gcc/src/type_.rs
+++ b/compiler/rustc_codegen_gcc/src/type_.rs
@@ -1,5 +1,3 @@
-use std::convert::TryInto;
-
 use gccjit::{RValue, Struct, Type};
 use rustc_codegen_ssa::traits::{BaseTypeMethods, DerivedTypeMethods, TypeMembershipMethods};
 use rustc_codegen_ssa::common::TypeKind;
@@ -202,8 +200,9 @@ impl<'gcc, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         value.get_type()
     }
 
-    fn type_array(&self, ty: Type<'gcc>, mut len: u64) -> Type<'gcc> {
-        if let Some(struct_type) = ty.is_struct() {
+    fn type_array(&self, ty: Type<'gcc>, len: u64) -> Type<'gcc> {
+        // TODO: remove this as well?
+        /*if let Some(struct_type) = ty.is_struct() {
             if struct_type.get_field_count() == 0 {
                 // NOTE: since gccjit only supports i32 for the array size and libcore's tests uses a
                 // size of usize::MAX in test_binary_search, we workaround this by setting the size to
@@ -211,14 +210,7 @@ impl<'gcc, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                 // FIXME(antoyo): fix gccjit API.
                 len = 0;
             }
-        }
-
-        // NOTE: see note above. Some other test uses usize::MAX.
-        if len == u64::MAX {
-            len = 0;
-        }
-
-        let len: i32 = len.try_into().expect("array len");
+        }*/
 
         self.context.new_array_type(None, ty, len)
     }
@@ -247,10 +239,6 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     pub fn type_named_struct(&self, name: &str) -> Struct<'gcc> {
         self.context.new_opaque_struct_type(None, name)
     }
-
-    pub fn type_bool(&self) -> Type<'gcc> {
-        self.context.new_type::<bool>()
-    }
 }
 
 pub fn struct_fields<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout<'tcx>) -> (Vec<Type<'gcc>>, bool) {
@@ -273,7 +261,7 @@ pub fn struct_fields<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout
         assert_eq!(offset.align_to(padding_align) + padding, target_offset);
         result.push(cx.type_padding_filler(padding, padding_align));
 
-        result.push(field.gcc_type(cx, !field.ty.is_any_ptr())); // FIXME(antoyo): might need to check if the type is inside another, like Box<Type>.
+        result.push(field.gcc_type(cx));
         offset = target_offset + field.size;
         prev_effective_align = effective_field_align;
     }
diff --git a/compiler/rustc_codegen_gcc/src/type_of.rs b/compiler/rustc_codegen_gcc/src/type_of.rs
index ea2ce7650..5df8c1a20 100644
--- a/compiler/rustc_codegen_gcc/src/type_of.rs
+++ b/compiler/rustc_codegen_gcc/src/type_of.rs
@@ -6,7 +6,7 @@ use rustc_middle::bug;
 use rustc_middle::ty::{self, Ty, TypeVisitableExt};
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
 use rustc_middle::ty::print::with_no_trimmed_paths;
-use rustc_target::abi::{self, Abi, F32, F64, FieldsShape, Int, Integer, Pointer, PointeeInfo, Size, TyAbiInterface, Variants};
+use rustc_target::abi::{self, Abi, Align, F32, F64, FieldsShape, Int, Integer, Pointer, PointeeInfo, Size, TyAbiInterface, Variants};
 use rustc_target::abi::call::{CastTarget, FnAbi, Reg};
 
 use crate::abi::{FnAbiGccExt, GccType};
@@ -50,11 +50,25 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 }
 
-pub fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout<'tcx>, defer: &mut Option<(Struct<'gcc>, TyAndLayout<'tcx>)>) -> Type<'gcc> {
+impl<'a, 'tcx> CodegenCx<'a, 'tcx> {
+    pub fn align_of(&self, ty: Ty<'tcx>) -> Align {
+        self.layout_of(ty).align.abi
+    }
+}
+
+fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout<'tcx>, defer: &mut Option<(Struct<'gcc>, TyAndLayout<'tcx>)>) -> Type<'gcc> {
     match layout.abi {
         Abi::Scalar(_) => bug!("handled elsewhere"),
         Abi::Vector { ref element, count } => {
             let element = layout.scalar_gcc_type_at(cx, element, Size::ZERO);
+            let element =
+                // NOTE: gcc doesn't allow pointer types in vectors.
+                if element.get_pointee().is_some() {
+                    cx.usize_type
+                }
+                else {
+                    element
+                };
             return cx.context.new_vector_type(element, count);
         },
         Abi::ScalarPair(..) => {
@@ -114,7 +128,7 @@ pub fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLa
                 },
             }
         }
-        FieldsShape::Array { count, .. } => cx.type_array(layout.field(cx, 0).gcc_type(cx, true), count),
+        FieldsShape::Array { count, .. } => cx.type_array(layout.field(cx, 0).gcc_type(cx), count),
         FieldsShape::Arbitrary { .. } =>
             match name {
                 None => {
@@ -133,7 +147,7 @@ pub fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLa
 pub trait LayoutGccExt<'tcx> {
     fn is_gcc_immediate(&self) -> bool;
     fn is_gcc_scalar_pair(&self) -> bool;
-    fn gcc_type<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>, set_fields: bool) -> Type<'gcc>;
+    fn gcc_type<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc>;
     fn immediate_gcc_type<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc>;
     fn scalar_gcc_type_at<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>, scalar: &abi::Scalar, offset: Size) -> Type<'gcc>;
     fn scalar_pair_element_gcc_type<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>, index: usize, immediate: bool) -> Type<'gcc>;
@@ -168,8 +182,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
     /// with the inner-most trailing unsized field using the "minimal unit"
     /// of that field's type - this is useful for taking the address of
     /// that field and ensuring the struct has the right alignment.
-    //TODO(antoyo): do we still need the set_fields parameter?
-    fn gcc_type<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>, set_fields: bool) -> Type<'gcc> {
+    fn gcc_type<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc> {
         if let Abi::Scalar(ref scalar) = self.abi {
             // Use a different cache for scalars because pointers to DSTs
             // can be either fat or thin (data pointers of fat pointers).
@@ -179,10 +192,10 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
             let ty =
                 match *self.ty.kind() {
                     ty::Ref(_, ty, _) | ty::RawPtr(ty::TypeAndMut { ty, .. }) => {
-                        cx.type_ptr_to(cx.layout_of(ty).gcc_type(cx, set_fields))
+                        cx.type_ptr_to(cx.layout_of(ty).gcc_type(cx))
                     }
                     ty::Adt(def, _) if def.is_box() => {
-                        cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).gcc_type(cx, true))
+                        cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).gcc_type(cx))
                     }
                     ty::FnPtr(sig) => cx.fn_ptr_backend_type(&cx.fn_abi_of_fn_ptr(sig, ty::List::empty())),
                     _ => self.scalar_gcc_type_at(cx, scalar, Size::ZERO),
@@ -199,13 +212,6 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
             };
         let cached_type = cx.types.borrow().get(&(self.ty, variant_index)).cloned();
         if let Some(ty) = cached_type {
-            let type_to_set_fields = cx.types_with_fields_to_set.borrow_mut().remove(&ty);
-            if let Some((struct_type, layout)) = type_to_set_fields {
-                // Since we might be trying to generate a type containing another type which is not
-                // completely generated yet, we deferred setting the fields until now.
-                let (fields, packed) = struct_fields(cx, layout);
-                cx.set_struct_body(struct_type, &fields, packed);
-            }
             return ty;
         }
 
@@ -222,7 +228,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
                 if let Some(v) = variant_index {
                     layout = layout.for_variant(cx, v);
                 }
-                layout.gcc_type(cx, true)
+                layout.gcc_type(cx)
             }
             else {
                 uncached_gcc_type(cx, *self, &mut defer)
@@ -230,9 +236,9 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
 
         cx.types.borrow_mut().insert((self.ty, variant_index), ty);
 
-        if let Some((ty, layout)) = defer {
+        if let Some((deferred_ty, layout)) = defer {
             let (fields, packed) = struct_fields(cx, layout);
-            cx.set_struct_body(ty, &fields, packed);
+            cx.set_struct_body(deferred_ty, &fields, packed);
         }
 
         ty
@@ -244,7 +250,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
                 return cx.type_i1();
             }
         }
-        self.gcc_type(cx, true)
+        self.gcc_type(cx)
     }
 
     fn scalar_gcc_type_at<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>, scalar: &abi::Scalar, offset: Size) -> Type<'gcc> {
@@ -273,7 +279,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
         // pointee types, to avoid bitcasting every `OperandRef::deref`.
         match self.ty.kind() {
             ty::Ref(..) | ty::RawPtr(_) => {
-                return self.field(cx, index).gcc_type(cx, true);
+                return self.field(cx, index).gcc_type(cx);
             }
             // only wide pointer boxes are handled as pointers
             // thin pointer boxes with scalar allocators are handled by the general logic below
@@ -343,7 +349,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
 
 impl<'gcc, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     fn backend_type(&self, layout: TyAndLayout<'tcx>) -> Type<'gcc> {
-        layout.gcc_type(self, true)
+        layout.gcc_type(self)
     }
 
     fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> Type<'gcc> {
diff --git a/compiler/rustc_codegen_gcc/test.sh b/compiler/rustc_codegen_gcc/test.sh
index c5ffb7636..6139892ae 100755
--- a/compiler/rustc_codegen_gcc/test.sh
+++ b/compiler/rustc_codegen_gcc/test.sh
@@ -17,17 +17,20 @@ export LIBRARY_PATH="$GCC_PATH"
 flags=
 gcc_master_branch=1
 channel="debug"
-func=all
+funcs=()
 build_only=0
+nb_parts=0
+current_part=0
 
 while [[ $# -gt 0 ]]; do
     case $1 in
         --release)
             codegen_channel=release
+            channel="release"
             shift
             ;;
         --release-sysroot)
-            sysroot_channel=release
+            sysroot_channel="--release"
             shift
             ;;
         --no-default-features)
@@ -40,43 +43,83 @@ while [[ $# -gt 0 ]]; do
             flags="$flags --features $1"
             shift
             ;;
-        --release)
-            channel="release"
+        "--test-rustc")
+            funcs+=(test_rustc)
             shift
             ;;
-        "--test-rustc")
-            func=test_rustc
+        "--test-successful-rustc")
+            funcs+=(test_successful_rustc)
+            shift
+            ;;
+        "--test-failing-rustc")
+            funcs+=(test_failing_rustc)
             shift
             ;;
 
         "--test-libcore")
-            func=test_libcore
+            funcs+=(test_libcore)
             shift
             ;;
 
         "--clean-ui-tests")
-            func=clean_ui_tests
+            funcs+=(clean_ui_tests)
+            shift
+            ;;
+        "--clean")
+            funcs+=(clean)
             shift
             ;;
 
         "--std-tests")
-            func=std_tests
+            funcs+=(std_tests)
+            shift
+            ;;
+
+        "--asm-tests")
+            funcs+=(asm_tests)
             shift
             ;;
 
         "--extended-tests")
-            func=extended_sysroot_tests
+            funcs+=(extended_sysroot_tests)
+            shift
+            ;;
+        "--extended-rand-tests")
+            funcs+=(extended_rand_tests)
+            shift
+            ;;
+        "--extended-regex-example-tests")
+            funcs+=(extended_regex_example_tests)
+            shift
+            ;;
+        "--extended-regex-tests")
+            funcs+=(extended_regex_tests)
+            shift
+            ;;
+
+        "--mini-tests")
+            funcs+=(mini_tests)
             shift
             ;;
 
         "--build-sysroot")
-            func=build_sysroot
+            funcs+=(build_sysroot)
             shift
             ;;
         "--build")
             build_only=1
             shift
             ;;
+        "--nb-parts")
+            shift
+            nb_parts=$1
+            shift
+            ;;
+        "--current-part")
+            shift
+            current_part=$1
+            shift
+            ;;
         *)
             echo "Unknown option $1"
             exit 1
@@ -87,7 +130,6 @@ done
 if [[ $channel == "release" ]]; then
     export CHANNEL='release'
     CARGO_INCREMENTAL=1 cargo rustc --release $flags
-    shift
 else
     echo $LD_LIBRARY_PATH
     export CHANNEL='debug'
@@ -95,6 +137,7 @@ else
 fi
 
 if (( $build_only == 1 )); then
+    echo "Since it's 'build-only', exiting..."
     exit
 fi
 
@@ -119,7 +162,7 @@ function mini_tests() {
 
 function build_sysroot() {
     echo "[BUILD] sysroot"
-    time ./build_sysroot/build_sysroot.sh
+    time ./build_sysroot/build_sysroot.sh $sysroot_channel
 }
 
 function std_tests() {
@@ -148,17 +191,57 @@ function std_tests() {
     $RUN_WRAPPER ./target/out/std_example --target $TARGET_TRIPLE
 
     echo "[AOT] subslice-patterns-const-eval"
-    $RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE
+    $RUSTC example/subslice-patterns-const-eval.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/subslice-patterns-const-eval
 
     echo "[AOT] track-caller-attribute"
-    $RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE
+    $RUSTC example/track-caller-attribute.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE
     $RUN_WRAPPER ./target/out/track-caller-attribute
 
     echo "[BUILD] mod_bench"
     $RUSTC example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE
 }
 
+function setup_rustc() {
+    rust_toolchain=$(cat rust-toolchain | grep channel | sed 's/channel = "\(.*\)"/\1/')
+
+    git clone https://github.com/rust-lang/rust.git || true
+    cd rust
+    git fetch
+    git checkout $(rustc -V | cut -d' ' -f3 | tr -d '(')
+    export RUSTFLAGS=
+
+    rm config.toml || true
+
+    cat > config.toml <<EOF
+[rust]
+codegen-backends = []
+deny-warnings = false
+
+[build]
+cargo = "$(which cargo)"
+local-rebuild = true
+rustc = "$HOME/.rustup/toolchains/$rust_toolchain-$TARGET_TRIPLE/bin/rustc"
+
+[target.x86_64-unknown-linux-gnu]
+llvm-filecheck = "`which FileCheck-10 || which FileCheck-11 || which FileCheck-12 || which FileCheck-13 || which FileCheck-14`"
+
+[llvm]
+download-ci-llvm = false
+EOF
+
+    rustc -V | cut -d' ' -f3 | tr -d '('
+    git checkout $(rustc -V | cut -d' ' -f3 | tr -d '(') tests
+}
+
+function asm_tests() {
+    setup_rustc
+
+    echo "[TEST] rustc test suite"
+    RUSTC_ARGS="-Zpanic-abort-tests -Csymbol-mangling-version=v0 -Zcodegen-backend="$(pwd)"/../target/"$CHANNEL"/librustc_codegen_gcc."$dylib_ext" --sysroot "$(pwd)"/../build_sysroot/sysroot -Cpanic=abort"
+    COMPILETEST_FORCE_STAGE0=1 ./x.py test --run always --stage 0 tests/assembly/asm --rustc-args "$RUSTC_ARGS"
+}
+
 # FIXME(antoyo): linker gives multiple definitions error on Linux
 #echo "[BUILD] sysroot in release mode"
 #./build_sysroot/build_sysroot.sh --release
@@ -187,7 +270,7 @@ function test_libcore() {
 #echo "[BENCH RUN] mod_bench"
 #hyperfine --runs ${RUN_RUNS:-10} ./target/out/mod_bench{,_inline} ./target/out/mod_bench_llvm_*
 
-function extended_sysroot_tests() {
+function extended_rand_tests() {
     if (( $gcc_master_branch == 0 )); then
         return
     fi
@@ -197,17 +280,12 @@ function extended_sysroot_tests() {
     echo "[TEST] rust-random/rand"
     ../cargo.sh test --workspace
     popd
+}
 
-    #pushd simple-raytracer
-    #echo "[BENCH COMPILE] ebobby/simple-raytracer"
-    #hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "cargo clean" \
-    #"RUSTC=rustc RUSTFLAGS='' cargo build" \
-    #"../cargo.sh build"
-
-    #echo "[BENCH RUN] ebobby/simple-raytracer"
-    #cp ./target/debug/main ./raytracer_cg_gcc
-    #hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_gcc
-    #popd
+function extended_regex_example_tests() {
+    if (( $gcc_master_branch == 0 )); then
+        return
+    fi
 
     pushd regex
     echo "[TEST] rust-lang/regex example shootout-regex-dna"
@@ -219,41 +297,43 @@ function extended_sysroot_tests() {
         | ../cargo.sh run --example shootout-regex-dna \
         | grep -v "Spawned thread" > res.txt
     diff -u res.txt examples/regexdna-output.txt
+    popd
+}
+
+function extended_regex_tests() {
+    if (( $gcc_master_branch == 0 )); then
+        return
+    fi
 
+    pushd regex
     echo "[TEST] rust-lang/regex tests"
+    export CG_RUSTFLAGS="--cap-lints warn" # newer aho_corasick versions throw a deprecation warning
     ../cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
     popd
 }
 
-function test_rustc() {
-    echo
-    echo "[TEST] rust-lang/rust"
-
-    rust_toolchain=$(cat rust-toolchain | grep channel | sed 's/channel = "\(.*\)"/\1/')
-
-    git clone https://github.com/rust-lang/rust.git || true
-    cd rust
-    git fetch
-    git checkout $(rustc -V | cut -d' ' -f3 | tr -d '(')
-    export RUSTFLAGS=
-
-    git apply ../rustc_patches/compile_test.patch || true
+function extended_sysroot_tests() {
+    #pushd simple-raytracer
+    #echo "[BENCH COMPILE] ebobby/simple-raytracer"
+    #hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "cargo clean" \
+    #"RUSTC=rustc RUSTFLAGS='' cargo build" \
+    #"../cargo.sh build"
 
-    rm config.toml || true
+    #echo "[BENCH RUN] ebobby/simple-raytracer"
+    #cp ./target/debug/main ./raytracer_cg_gcc
+    #hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_gcc
+    #popd
 
-    cat > config.toml <<EOF
-[rust]
-codegen-backends = []
-deny-warnings = false
+    extended_rand_tests
+    extended_regex_example_tests
+    extended_regex_tests
+}
 
-[build]
-cargo = "$(which cargo)"
-local-rebuild = true
-rustc = "$HOME/.rustup/toolchains/$rust_toolchain-$TARGET_TRIPLE/bin/rustc"
-EOF
+function test_rustc() {
+    echo
+    echo "[TEST] rust-lang/rust"
 
-    rustc -V | cut -d' ' -f3 | tr -d '('
-    git checkout $(rustc -V | cut -d' ' -f3 | tr -d '(') tests
+    setup_rustc
 
     for test in $(rg -i --files-with-matches "//(\[\w+\])?~|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" tests/ui); do
       rm $test
@@ -261,21 +341,61 @@ EOF
 
     git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
 
-    rm -r tests/ui/{abi*,extern/,panic-runtime/,panics/,unsized-locals/,proc-macro/,threads-sendsync/,thinlto/,borrowck/,test*,*lto*.rs} || true
-    for test in $(rg --files-with-matches "catch_unwind|should_panic|thread|lto" tests/ui); do
+    rm -r tests/ui/{abi*,extern/,unsized-locals/,proc-macro/,threads-sendsync/,thinlto/,borrowck/,chalkify/bugs/,test*,*lto*.rs,consts/const-float-bits-reject-conv.rs,consts/issue-miri-1910.rs} || true
+    rm tests/ui/mir/mir_heavy_promoted.rs # this tests is oom-killed in the CI.
+    for test in $(rg --files-with-matches "thread|lto" tests/ui); do
       rm $test
     done
+    git checkout tests/ui/lto/auxiliary/dylib.rs
     git checkout tests/ui/type-alias-impl-trait/auxiliary/cross_crate_ice.rs
     git checkout tests/ui/type-alias-impl-trait/auxiliary/cross_crate_ice2.rs
+    git checkout tests/ui/macros/rfc-2011-nicer-assert-messages/auxiliary/common.rs
+
+    RUSTC_ARGS="$TEST_FLAGS -Csymbol-mangling-version=v0 -Zcodegen-backend="$(pwd)"/../target/"$CHANNEL"/librustc_codegen_gcc."$dylib_ext" --sysroot "$(pwd)"/../build_sysroot/sysroot"
+
+    if [ $# -eq 0 ]; then
+        # No argument supplied to the function. Doing nothing.
+        echo "No argument provided. Keeping all UI tests"
+    elif [ $1 = "0" ]; then
+        # Removing the failing tests.
+        xargs -a ../failing-ui-tests.txt -d'\n' rm
+    else
+        # Removing all tests.
+        find tests/ui -type f -name '*.rs' -not -path '*/auxiliary/*' -delete
+        # Putting back only the failing ones.
+        xargs -a ../failing-ui-tests.txt -d'\n' git checkout --
+    fi
 
-    RUSTC_ARGS="-Zpanic-abort-tests -Csymbol-mangling-version=v0 -Zcodegen-backend="$(pwd)"/../target/"$CHANNEL"/librustc_codegen_gcc."$dylib_ext" --sysroot "$(pwd)"/../build_sysroot/sysroot -Cpanic=abort"
+    if [ $nb_parts -gt 0 ]; then
+        echo "Splitting ui_test into $nb_parts parts (and running part $current_part)"
+        find tests/ui -type f -name '*.rs' -not -path "*/auxiliary/*" > ui_tests
+        # To ensure it'll be always the same sub files, we sort the content.
+        sort ui_tests -o ui_tests
+        count=$((`wc -l < ui_tests` / $nb_parts))
+        # We increment the number of tests by one because if this is an odd number, we would skip
+        # one test.
+        count=$((count + 1))
+        split -d -l $count -a 1 ui_tests ui_tests.split
+        # Removing all tests.
+        find tests/ui -type f -name '*.rs' -not -path "*/auxiliary/*" -delete
+        # Putting back only the ones we want to test.
+        xargs -a "ui_tests.split$current_part" -d'\n' git checkout --
+    fi
 
     echo "[TEST] rustc test suite"
     COMPILETEST_FORCE_STAGE0=1 ./x.py test --run always --stage 0 tests/ui/ --rustc-args "$RUSTC_ARGS"
 }
 
+function test_failing_rustc() {
+    test_rustc "1"
+}
+
+function test_successful_rustc() {
+    test_rustc "0"
+}
+
 function clean_ui_tests() {
-    find rust/build/x86_64-unknown-linux-gnu/test/ui/ -name stamp -exec rm -rf {} \;
+    find rust/build/x86_64-unknown-linux-gnu/test/ui/ -name stamp -delete
 }
 
 function all() {
@@ -283,9 +403,17 @@ function all() {
     mini_tests
     build_sysroot
     std_tests
+    #asm_tests
     test_libcore
     extended_sysroot_tests
     test_rustc
 }
 
-$func
+if [ ${#funcs[@]} -eq 0 ]; then
+    echo "No command passed, running '--all'..."
+    all
+else
+    for t in ${funcs[@]}; do
+        $t
+    done
+fi
diff --git a/compiler/rustc_codegen_gcc/tests/lang_tests_common.rs b/compiler/rustc_codegen_gcc/tests/lang_tests_common.rs
index 8e378177e..06de26f7e 100644
--- a/compiler/rustc_codegen_gcc/tests/lang_tests_common.rs
+++ b/compiler/rustc_codegen_gcc/tests/lang_tests_common.rs
@@ -46,11 +46,15 @@ pub fn main_inner(profile: Profile) {
                 &format!("-Zcodegen-backend={}/target/debug/librustc_codegen_gcc.so", current_dir),
                 "--sysroot", &format!("{}/build_sysroot/sysroot/", current_dir),
                 "-Zno-parallel-llvm",
-                "-C", "panic=abort",
                 "-C", "link-arg=-lc",
                 "-o", exe.to_str().expect("to_str"),
                 path.to_str().expect("to_str"),
             ]);
+            if let Some(flags) = option_env!("TEST_FLAGS") {
+                for flag in flags.split_whitespace() {
+                    compiler.arg(&flag);
+                }
+            }
             match profile {
                 Profile::Debug => {}
                 Profile::Release => {
diff --git a/compiler/rustc_codegen_gcc/tests/run/abort1.rs b/compiler/rustc_codegen_gcc/tests/run/abort1.rs
index 291af5993..25041d93e 100644
--- a/compiler/rustc_codegen_gcc/tests/run/abort1.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/abort1.rs
@@ -33,6 +33,7 @@ mod intrinsics {
     use super::Sized;
 
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
diff --git a/compiler/rustc_codegen_gcc/tests/run/abort2.rs b/compiler/rustc_codegen_gcc/tests/run/abort2.rs
index 3c87c5678..e7443c8db 100644
--- a/compiler/rustc_codegen_gcc/tests/run/abort2.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/abort2.rs
@@ -33,6 +33,7 @@ mod intrinsics {
     use super::Sized;
 
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
diff --git a/compiler/rustc_codegen_gcc/tests/run/array.rs b/compiler/rustc_codegen_gcc/tests/run/array.rs
index 8b621d8a3..49b28d98f 100644
--- a/compiler/rustc_codegen_gcc/tests/run/array.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/array.rs
@@ -79,7 +79,7 @@ pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
 #[lang = "panic"]
 #[track_caller]
 #[no_mangle]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
     unsafe {
         libc::puts("Panicking\0" as *const str as *const u8);
         intrinsics::abort();
@@ -105,6 +105,7 @@ fn panic_bounds_check(index: usize, len: usize) -> ! {
 
 mod intrinsics {
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
diff --git a/compiler/rustc_codegen_gcc/tests/run/assign.rs b/compiler/rustc_codegen_gcc/tests/run/assign.rs
index eb38a8a38..427c1a250 100644
--- a/compiler/rustc_codegen_gcc/tests/run/assign.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/assign.rs
@@ -57,6 +57,7 @@ mod libc {
 
 mod intrinsics {
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
@@ -64,7 +65,7 @@ mod intrinsics {
 #[lang = "panic"]
 #[track_caller]
 #[no_mangle]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
     unsafe {
         libc::puts("Panicking\0" as *const str as *const u8);
         libc::fflush(libc::stdout);
diff --git a/compiler/rustc_codegen_gcc/tests/run/closure.rs b/compiler/rustc_codegen_gcc/tests/run/closure.rs
index 7121a5f0d..8daa681ab 100644
--- a/compiler/rustc_codegen_gcc/tests/run/closure.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/closure.rs
@@ -97,10 +97,14 @@ fn panic_bounds_check(index: usize, len: usize) -> ! {
 
 mod intrinsics {
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
 
+#[lang = "tuple_trait"]
+pub trait Tuple {}
+
 #[lang = "unsize"]
 pub trait Unsize<T: ?Sized> {}
 
@@ -114,7 +118,7 @@ impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<*mut U> for *mut T {}
 
 #[lang = "fn_once"]
 #[rustc_paren_sugar]
-pub trait FnOnce<Args> {
+pub trait FnOnce<Args: Tuple> {
     #[lang = "fn_once_output"]
     type Output;
 
@@ -123,7 +127,7 @@ pub trait FnOnce<Args> {
 
 #[lang = "fn_mut"]
 #[rustc_paren_sugar]
-pub trait FnMut<Args>: FnOnce<Args> {
+pub trait FnMut<Args: Tuple>: FnOnce<Args> {
     extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output;
 }
 
@@ -177,7 +181,7 @@ impl Add for isize {
 #[lang = "panic"]
 #[track_caller]
 #[no_mangle]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
     unsafe {
         libc::puts("Panicking\0" as *const str as *const u8);
         intrinsics::abort();
diff --git a/compiler/rustc_codegen_gcc/tests/run/condition.rs b/compiler/rustc_codegen_gcc/tests/run/condition.rs
index 6a2e2d5bb..b7a13081d 100644
--- a/compiler/rustc_codegen_gcc/tests/run/condition.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/condition.rs
@@ -82,7 +82,7 @@ pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
 #[lang = "panic"]
 #[track_caller]
 #[no_mangle]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
     unsafe {
         libc::puts("Panicking\0" as *const str as *const u8);
         intrinsics::abort();
@@ -108,6 +108,7 @@ fn panic_bounds_check(index: usize, len: usize) -> ! {
 
 mod intrinsics {
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
diff --git a/compiler/rustc_codegen_gcc/tests/run/fun_ptr.rs b/compiler/rustc_codegen_gcc/tests/run/fun_ptr.rs
index a226fff79..8a196f774 100644
--- a/compiler/rustc_codegen_gcc/tests/run/fun_ptr.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/fun_ptr.rs
@@ -76,7 +76,7 @@ pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
 #[lang = "panic"]
 #[track_caller]
 #[no_mangle]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
     unsafe {
         libc::puts("Panicking\0" as *const str as *const u8);
         intrinsics::abort();
@@ -102,6 +102,7 @@ fn panic_bounds_check(index: usize, len: usize) -> ! {
 
 mod intrinsics {
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
diff --git a/compiler/rustc_codegen_gcc/tests/run/int.rs b/compiler/rustc_codegen_gcc/tests/run/int.rs
index 75779622b..bfe73c384 100644
--- a/compiler/rustc_codegen_gcc/tests/run/int.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/int.rs
@@ -3,22 +3,14 @@
 // Run-time:
 //   status: 0
 
-#![feature(const_black_box, core_intrinsics, start)]
-
-#![no_std]
-
-#[panic_handler]
-fn panic_handler(_: &core::panic::PanicInfo) -> ! {
-    core::intrinsics::abort();
-}
+#![feature(const_black_box)]
 
 /*
  * Code
  */
 
-#[start]
-fn main(_argc: isize, _argv: *const *const u8) -> isize {
-    use core::hint::black_box;
+fn main() {
+    use std::hint::black_box;
 
     macro_rules! check {
         ($ty:ty, $expr:expr) => {
@@ -335,6 +327,4 @@ fn main(_argc: isize, _argv: *const *const u8) -> isize {
         const VAL5: T = 73236519889708027473620326106273939584_i128;
         check_ops128!();
     }
-
-    0
 }
diff --git a/compiler/rustc_codegen_gcc/tests/run/int_overflow.rs b/compiler/rustc_codegen_gcc/tests/run/int_overflow.rs
index ea2c5add9..c3fcb3c0a 100644
--- a/compiler/rustc_codegen_gcc/tests/run/int_overflow.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/int_overflow.rs
@@ -55,6 +55,7 @@ mod libc {
 
 mod intrinsics {
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
@@ -62,7 +63,7 @@ mod intrinsics {
 #[lang = "panic"]
 #[track_caller]
 #[no_mangle]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
     unsafe {
         // Panicking is expected iff overflow checking is enabled.
         #[cfg(debug_assertions)]
diff --git a/compiler/rustc_codegen_gcc/tests/run/mut_ref.rs b/compiler/rustc_codegen_gcc/tests/run/mut_ref.rs
index 52de20021..2a2ea8b8b 100644
--- a/compiler/rustc_codegen_gcc/tests/run/mut_ref.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/mut_ref.rs
@@ -59,6 +59,7 @@ mod libc {
 
 mod intrinsics {
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
@@ -66,7 +67,7 @@ mod intrinsics {
 #[lang = "panic"]
 #[track_caller]
 #[no_mangle]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
     unsafe {
         libc::puts("Panicking\0" as *const str as *const u8);
         libc::fflush(libc::stdout);
diff --git a/compiler/rustc_codegen_gcc/tests/run/operations.rs b/compiler/rustc_codegen_gcc/tests/run/operations.rs
index e078b37b4..67b9f241d 100644
--- a/compiler/rustc_codegen_gcc/tests/run/operations.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/operations.rs
@@ -65,6 +65,7 @@ mod libc {
 
 mod intrinsics {
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
@@ -72,7 +73,7 @@ mod intrinsics {
 #[lang = "panic"]
 #[track_caller]
 #[no_mangle]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
     unsafe {
         libc::puts("Panicking\0" as *const str as *const u8);
         libc::fflush(libc::stdout);
diff --git a/compiler/rustc_codegen_gcc/tests/run/ptr_cast.rs b/compiler/rustc_codegen_gcc/tests/run/ptr_cast.rs
index 6ac099ea1..da8a8295d 100644
--- a/compiler/rustc_codegen_gcc/tests/run/ptr_cast.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/ptr_cast.rs
@@ -76,7 +76,7 @@ pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
 #[lang = "panic"]
 #[track_caller]
 #[no_mangle]
-pub fn panic(_msg: &str) -> ! {
+pub fn panic(_msg: &'static str) -> ! {
     unsafe {
         libc::puts("Panicking\0" as *const str as *const u8);
         intrinsics::abort();
@@ -102,6 +102,7 @@ fn panic_bounds_check(index: usize, len: usize) -> ! {
 
 mod intrinsics {
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
diff --git a/compiler/rustc_codegen_gcc/tests/run/slice.rs b/compiler/rustc_codegen_gcc/tests/run/slice.rs
index ad9258ed0..96f1c4792 100644
--- a/compiler/rustc_codegen_gcc/tests/run/slice.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/slice.rs
@@ -102,6 +102,7 @@ mod intrinsics {
     use super::Sized;
 
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
diff --git a/compiler/rustc_codegen_gcc/tests/run/static.rs b/compiler/rustc_codegen_gcc/tests/run/static.rs
index 294add968..19201f1df 100644
--- a/compiler/rustc_codegen_gcc/tests/run/static.rs
+++ b/compiler/rustc_codegen_gcc/tests/run/static.rs
@@ -45,6 +45,7 @@ mod intrinsics {
     use super::Sized;
 
     extern "rust-intrinsic" {
+        #[rustc_safe_intrinsic]
         pub fn abort() -> !;
     }
 }
diff --git a/compiler/rustc_codegen_gcc/tools/check_intrinsics_duplicates.py b/compiler/rustc_codegen_gcc/tools/check_intrinsics_duplicates.py
new file mode 100644
index 000000000..c09fb3c75
--- /dev/null
+++ b/compiler/rustc_codegen_gcc/tools/check_intrinsics_duplicates.py
@@ -0,0 +1,67 @@
+import sys
+
+
+def check_duplicates():
+    auto_content = ""
+    manual_content = ""
+
+    with open("src/intrinsic/llvm.rs", "r", encoding="utf8") as f:
+        manual_content = f.read()
+    with open("src/intrinsic/archs.rs", "r", encoding="utf8") as f:
+        auto_content = f.read()
+
+    intrinsics_map = {}
+    for line in auto_content.splitlines():
+        line = line.strip()
+        if not line.startswith('"'):
+            continue
+        parts = line.split('"')
+        if len(parts) != 5:
+            continue
+        intrinsics_map[parts[1]] = parts[3]
+
+    if len(intrinsics_map) == 0:
+        print("No intrinsics found in auto code... Aborting.")
+        return 1
+    print("Found {} intrinsics in auto code".format(len(intrinsics_map)))
+    errors = []
+    lines = manual_content.splitlines()
+    pos = 0
+    found = 0
+    while pos < len(lines):
+        line = lines[pos].strip()
+        # This is our marker.
+        if line == "let gcc_name = match name {":
+            while pos < len(lines):
+                line = lines[pos].strip()
+                pos += 1
+                if line == "};":
+                    # We're done!
+                    if found == 0:
+                        print("No intrinsics found in manual code even though we found the "
+                            "marker... Aborting...")
+                        return 1
+                    for error in errors:
+                        print("ERROR => {}".format(error))
+                    return 1 if len(errors) != 0 else 0
+                parts = line.split('"')
+                if len(parts) != 5:
+                    continue
+                found += 1
+                if parts[1] in intrinsics_map:
+                    if parts[3] != intrinsics_map[parts[1]]:
+                        print("Same intrinsics (`{}` at line {}) but different GCC "
+                            "translations: `{}` != `{}`".format(
+                                parts[1], pos, intrinsics_map[parts[1]], parts[3]))
+                    else:
+                        errors.append("Duplicated intrinsics: `{}` at line {}. Please remove it "
+                            " from manual code".format(parts[1], pos))
+            # Weird but whatever...
+            return 1 if len(errors) != 0 else 0
+        pos += 1
+    print("No intrinsics found in manual code... Aborting")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(check_duplicates())
diff --git a/compiler/rustc_codegen_gcc/tools/generate_intrinsics.py b/compiler/rustc_codegen_gcc/tools/generate_intrinsics.py
index 849c6e9c9..6188924b0 100644
--- a/compiler/rustc_codegen_gcc/tools/generate_intrinsics.py
+++ b/compiler/rustc_codegen_gcc/tools/generate_intrinsics.py
@@ -13,7 +13,7 @@ def run_command(command, cwd=None):
         sys.exit(1)
 
 
-def clone_repository(repo_name, path, repo_url, sub_path=None):
+def clone_repository(repo_name, path, repo_url, sub_paths=None):
     if os.path.exists(path):
         while True:
             choice = input("There is already a `{}` folder, do you want to update it? [y/N]".format(path))
@@ -27,12 +27,12 @@ def clone_repository(repo_name, path, repo_url, sub_path=None):
             else:
                 print("Didn't understand answer...")
     print("Cloning {} repository...".format(repo_name))
-    if sub_path is None:
+    if sub_paths is None:
         run_command(["git", "clone", repo_url, "--depth", "1", path])
     else:
         run_command(["git", "clone", repo_url, "--filter=tree:0", "--no-checkout", path])
         run_command(["git", "sparse-checkout", "init"], cwd=path)
-        run_command(["git", "sparse-checkout", "set", "add", sub_path], cwd=path)
+        run_command(["git", "sparse-checkout", "set", *sub_paths], cwd=path)
         run_command(["git", "checkout"], cwd=path)
 
 
@@ -40,56 +40,45 @@ def append_intrinsic(array, intrinsic_name, translation):
     array.append((intrinsic_name, translation))
 
 
-def extract_instrinsics(intrinsics, file):
-    print("Extracting intrinsics from `{}`...".format(file))
-    with open(file, "r", encoding="utf8") as f:
-        content = f.read()
+def convert_to_string(content):
+    if content.__class__.__name__ == 'bytes':
+        return content.decode('utf-8')
+    return content
 
-    lines = content.splitlines()
+
+def extract_instrinsics_from_llvm(llvm_path, intrinsics):
+    p = subprocess.Popen(
+        ["llvm-tblgen", "llvm/IR/Intrinsics.td"],
+        cwd=os.path.join(llvm_path, "llvm/include"),
+        stdout=subprocess.PIPE)
+    output, err = p.communicate()
+    lines = convert_to_string(output).splitlines()
     pos = 0
-    current_arch = None
     while pos < len(lines):
-        line = lines[pos].strip()
-        if line.startswith("let TargetPrefix ="):
-            current_arch = line.split('"')[1].strip()
-            if len(current_arch) == 0:
-                current_arch = None
-        elif current_arch is None:
-            pass
-        elif line == "}":
-            current_arch = None
-        elif line.startswith("def "):
-            content = ""
-            while not content.endswith(";") and not content.endswith("}") and pos < len(lines):
-                line = lines[pos].split(" // ")[0].strip()
-                content += line
-                pos += 1
-            entries = re.findall('GCCBuiltin<"(\\w+)">', content)
-            if len(entries) > 0:
-                intrinsic = content.split("def ")[1].strip().split(":")[0].strip()
-                intrinsic = intrinsic.split("_")
-                if len(intrinsic) < 2 or intrinsic[0] != "int":
-                    continue
-                intrinsic[0] = "llvm"
-                intrinsic = ".".join(intrinsic)
-                if current_arch not in intrinsics:
-                    intrinsics[current_arch] = []
-                for entry in entries:
-                    append_intrinsic(intrinsics[current_arch], intrinsic, entry)
+        line = lines[pos]
+        if not line.startswith("def "):
+            pos += 1
             continue
-        pos += 1
-        continue
-    print("Done!")
-
-
-def extract_instrinsics_from_llvm(llvm_path, intrinsics):
-    files = []
-    intrinsics_path = os.path.join(llvm_path, "llvm/include/llvm/IR")
-    for (dirpath, dirnames, filenames) in walk(intrinsics_path):
-        files.extend([os.path.join(intrinsics_path, f) for f in filenames if f.endswith(".td")])
-
-    for file in files:
-        extract_instrinsics(intrinsics, file)
+        intrinsic = line.split(" ")[1].strip()
+        content = line
+        while pos < len(lines):
+            line = lines[pos].split(" // ")[0].strip()
+            content += line
+            pos += 1
+            if line == "}":
+                break
+        entries = re.findall('string ClangBuiltinName = "(\\w+)";', content)
+        current_arch = re.findall('string TargetPrefix = "(\\w+)";', content)
+        if len(entries) == 1 and len(current_arch) == 1:
+            current_arch = current_arch[0]
+            intrinsic = intrinsic.split("_")
+            if len(intrinsic) < 2 or intrinsic[0] != "int":
+                continue
+            intrinsic[0] = "llvm"
+            intrinsic = ".".join(intrinsic)
+            if current_arch not in intrinsics:
+                intrinsics[current_arch] = []
+            append_intrinsic(intrinsics[current_arch], intrinsic, entries[0])
 
 
 def append_translation(json_data, p, array):
@@ -193,6 +182,8 @@ def update_intrinsics(llvm_path, llvmint, llvmint2):
             for entry in intrinsics[arch]:
                 if entry[2] == True: # if it is a duplicate
                     out.write('    // [DUPLICATE]: "{}" => "{}",\n'.format(entry[0], entry[1]))
+                elif "_round_mask" in entry[1]:
+                    out.write('    // [INVALID CONVERSION]: "{}" => "{}",\n'.format(entry[0], entry[1]))
                 else:
                     out.write('    "{}" => "{}",\n'.format(entry[0], entry[1]))
         out.write('    _ => unimplemented!("***** unsupported LLVM intrinsic {}", name),\n')
@@ -219,7 +210,7 @@ def main():
         "llvm-project",
         llvm_path,
         "https://github.com/llvm/llvm-project",
-        sub_path="llvm/include/llvm/IR",
+        sub_paths=["llvm/include/llvm/IR", "llvm/include/llvm/CodeGen/"],
     )
     clone_repository(
         "llvmint",
diff --git a/compiler/rustc_codegen_llvm/locales/en-US.ftl b/compiler/rustc_codegen_llvm/messages.ftl
index e5df41737..b6d7484bc 100644
--- a/compiler/rustc_codegen_llvm/locales/en-US.ftl
+++ b/compiler/rustc_codegen_llvm/messages.ftl
@@ -27,7 +27,8 @@ codegen_llvm_error_calling_dlltool =
     Error calling dlltool: {$error}
 
 codegen_llvm_dlltool_fail_import_library =
-    Dlltool could not create import library: {$stdout}\n{$stderr}
+    Dlltool could not create import library: {$stdout}
+    {$stderr}
 
 codegen_llvm_target_feature_disable_or_enable =
     the target features {$features} must all be either enabled or disabled together
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index d9f8170a3..1a3865360 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -244,6 +244,9 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 InlineAsmArch::Msp430 => {
                     constraints.push("~{sr}".to_string());
                 }
+                InlineAsmArch::M68k => {
+                    constraints.push("~{ccr}".to_string());
+                }
             }
         }
         if !options.contains(InlineAsmOptions::NOMEM) {
@@ -381,7 +384,7 @@ impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> {
         }
 
         unsafe {
-            llvm::LLVMRustAppendModuleInlineAsm(
+            llvm::LLVMAppendModuleInlineAsm(
                 self.llmod,
                 template_str.as_ptr().cast(),
                 template_str.len(),
@@ -671,6 +674,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r",
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
             InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a",
+            InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d",
             InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
                 bug!("LLVM backend does not support SPIR-V")
             }
@@ -768,6 +774,7 @@ fn modifier_to_llvm(
         InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
             bug!("LLVM backend does not support SPIR-V")
         }
+        InlineAsmRegClass::M68k(_) => None,
         InlineAsmRegClass::Err => unreachable!(),
     }
 }
@@ -839,6 +846,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'
         InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(),
         InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
         InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
+        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
+        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
+        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
         InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
             bug!("LLVM backend does not support SPIR-V")
         }
diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs
index dd3268d77..12da21dc4 100644
--- a/compiler/rustc_codegen_llvm/src/back/archive.rs
+++ b/compiler/rustc_codegen_llvm/src/back/archive.rs
@@ -110,7 +110,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
     fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> {
         // FIXME use ArArchiveBuilder on most targets again once reading thin archives is
         // implemented
-        if true || sess.target.arch == "wasm32" || sess.target.arch == "wasm64" {
+        if true {
             Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() })
         } else {
             Box::new(ArArchiveBuilder::new(sess, get_llvm_object_symbols))
@@ -189,6 +189,15 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
                 path.push(lib_name);
                 path
             };
+            // dlltool target architecture args from:
+            // https://github.com/llvm/llvm-project-release-prs/blob/llvmorg-15.0.6/llvm/lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp#L69
+            let (dlltool_target_arch, dlltool_target_bitness) = match sess.target.arch.as_ref() {
+                "x86_64" => ("i386:x86-64", "--64"),
+                "x86" => ("i386", "--32"),
+                "aarch64" => ("arm64", "--64"),
+                "arm" => ("arm", "--32"),
+                _ => panic!("unsupported arch {}", sess.target.arch),
+            };
             let result = std::process::Command::new(dlltool)
                 .args([
                     "-d",
@@ -197,6 +206,10 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
                     lib_name,
                     "-l",
                     output_path.to_str().unwrap(),
+                    "-m",
+                    dlltool_target_arch,
+                    "-f",
+                    dlltool_target_bitness,
                     "--no-leading-underscore",
                     "--temp-prefix",
                     temp_prefix.to_str().unwrap(),
@@ -422,24 +435,22 @@ fn find_binutils_dlltool(sess: &Session) -> OsString {
         return dlltool_path.clone().into_os_string();
     }
 
-    let mut tool_name: OsString = if sess.host.arch != sess.target.arch {
-        // We are cross-compiling, so we need the tool with the prefix matching our target
-        if sess.target.arch == "x86" {
-            "i686-w64-mingw32-dlltool"
-        } else {
-            "x86_64-w64-mingw32-dlltool"
-        }
+    let tool_name: OsString = if sess.host.options.is_like_windows {
+        // If we're compiling on Windows, always use "dlltool.exe".
+        "dlltool.exe"
     } else {
-        // We are not cross-compiling, so we just want `dlltool`
-        "dlltool"
+        // On other platforms, use the architecture-specific name.
+        match sess.target.arch.as_ref() {
+            "x86_64" => "x86_64-w64-mingw32-dlltool",
+            "x86" => "i686-w64-mingw32-dlltool",
+            "aarch64" => "aarch64-w64-mingw32-dlltool",
+
+            // For non-standard architectures (e.g., aarch32) fallback to "dlltool".
+            _ => "dlltool",
+        }
     }
     .into();
 
-    if sess.host.options.is_like_windows {
-        // If we're compiling on Windows, add the .exe suffix
-        tool_name.push(".exe");
-    }
-
     // NOTE: it's not clear how useful it is to explicitly search PATH.
     for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) {
         let full_path = dir.join(&tool_name);
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index a4ae1b01e..7136f750f 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -214,6 +214,8 @@ pub fn target_machine_factory(
 
     let path_mapping = sess.source_map().path_mapping().clone();
 
+    let force_emulated_tls = sess.target.force_emulated_tls;
+
     Arc::new(move |config: TargetMachineFactoryConfig| {
         let split_dwarf_file =
             path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0;
@@ -239,6 +241,7 @@ pub fn target_machine_factory(
                 relax_elf_relocations,
                 use_init_array,
                 split_dwarf_file.as_ptr(),
+                force_emulated_tls,
             )
         };
 
@@ -901,9 +904,9 @@ unsafe fn embed_bitcode(
         // We need custom section flags, so emit module-level inline assembly.
         let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
         let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
-        llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len());
+        llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len());
         let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes());
-        llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len());
+        llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len());
     }
 }
 
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index 0f33b9854..6819a2af0 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -2,7 +2,7 @@ use crate::abi::FnAbiLlvmExt;
 use crate::attributes;
 use crate::common::Funclet;
 use crate::context::CodegenCx;
-use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock};
+use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True};
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -841,7 +841,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
-        unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) }
+        unsafe {
+            llvm::LLVMBuildIntCast2(
+                self.llbuilder,
+                val,
+                dest_ty,
+                if is_signed { True } else { False },
+                UNNAMED,
+            )
+        }
     }
 
     fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
@@ -990,7 +998,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
         let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false);
-        let mut exn = self.const_undef(ty);
+        let mut exn = self.const_poison(ty);
         exn = self.insert_value(exn, exn0, 0);
         exn = self.insert_value(exn, exn1, 1);
         unsafe {
@@ -1001,11 +1009,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
         let name = cstr!("cleanuppad");
         let ret = unsafe {
-            llvm::LLVMRustBuildCleanupPad(
+            llvm::LLVMBuildCleanupPad(
                 self.llbuilder,
                 parent,
-                args.len() as c_uint,
                 args.as_ptr(),
+                args.len() as c_uint,
                 name.as_ptr(),
             )
         };
@@ -1014,7 +1022,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
         unsafe {
-            llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
+            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
                 .expect("LLVM does not have support for cleanupret");
         }
     }
@@ -1022,11 +1030,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
         let name = cstr!("catchpad");
         let ret = unsafe {
-            llvm::LLVMRustBuildCatchPad(
+            llvm::LLVMBuildCatchPad(
                 self.llbuilder,
                 parent,
-                args.len() as c_uint,
                 args.as_ptr(),
+                args.len() as c_uint,
                 name.as_ptr(),
             )
         };
@@ -1041,7 +1049,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     ) -> &'ll Value {
         let name = cstr!("catchswitch");
         let ret = unsafe {
-            llvm::LLVMRustBuildCatchSwitch(
+            llvm::LLVMBuildCatchSwitch(
                 self.llbuilder,
                 parent,
                 unwind,
@@ -1052,7 +1060,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         let ret = ret.expect("LLVM does not have support for catchswitch");
         for handler in handlers {
             unsafe {
-                llvm::LLVMRustAddHandler(ret, handler);
+                llvm::LLVMAddHandler(ret, handler);
             }
         }
         ret
@@ -1190,8 +1198,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         // Set KCFI operand bundle
         let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
         let kcfi_bundle =
-            if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call {
-                let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap());
+            if let Some(fn_abi) = fn_abi && self.tcx.sess.is_sanitizer_kcfi_enabled() && is_indirect_call {
+                let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi);
                 Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
             } else {
                 None
@@ -1376,8 +1384,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
     }
 
     pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value {
-        let ret =
-            unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
+        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
         ret.expect("LLVM does not have support for catchret")
     }
 
diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs
index b0a9a30ab..4f8b5abd9 100644
--- a/compiler/rustc_codegen_llvm/src/common.rs
+++ b/compiler/rustc_codegen_llvm/src/common.rs
@@ -130,6 +130,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         unsafe { llvm::LLVMGetUndef(t) }
     }
 
+    fn const_poison(&self, t: &'ll Type) -> &'ll Value {
+        unsafe { llvm::LLVMGetPoison(t) }
+    }
+
     fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value {
         unsafe { llvm::LLVMConstInt(t, i as u64, True) }
     }
@@ -374,8 +378,7 @@ pub(crate) fn get_dllimport<'tcx>(
     name: &str,
 ) -> Option<&'tcx DllImport> {
     tcx.native_library(id)
-        .map(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name))
-        .flatten()
+        .and_then(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name))
 }
 
 pub(crate) fn is_mingw_gnu_toolchain(target: &Target) -> bool {
diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index 9116e71be..940358acd 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -7,7 +7,6 @@ use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
 use cstr::cstr;
-use libc::c_uint;
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
@@ -486,10 +485,10 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
             // go into custom sections of the wasm executable.
             if self.tcx.sess.target.is_like_wasm {
                 if let Some(section) = attrs.link_section {
-                    let section = llvm::LLVMMDStringInContext(
+                    let section = llvm::LLVMMDStringInContext2(
                         self.llcx,
                         section.as_str().as_ptr().cast(),
-                        section.as_str().len() as c_uint,
+                        section.as_str().len(),
                     );
                     assert!(alloc.provenance().ptrs().is_empty());
 
@@ -498,17 +497,15 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                     // as part of the interpreter execution).
                     let bytes =
                         alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len());
-                    let alloc = llvm::LLVMMDStringInContext(
-                        self.llcx,
-                        bytes.as_ptr().cast(),
-                        bytes.len() as c_uint,
-                    );
+                    let alloc =
+                        llvm::LLVMMDStringInContext2(self.llcx, bytes.as_ptr().cast(), bytes.len());
                     let data = [section, alloc];
-                    let meta = llvm::LLVMMDNodeInContext(self.llcx, data.as_ptr(), 2);
+                    let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len());
+                    let val = llvm::LLVMMetadataAsValue(self.llcx, meta);
                     llvm::LLVMAddNamedMetadataOperand(
                         self.llmod,
                         "wasm.custom_sections\0".as_ptr().cast(),
-                        meta,
+                        val,
                     );
                 }
             } else {
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index 3d29968d5..f0d729d47 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -735,9 +735,13 @@ impl<'ll> CodegenCx<'ll, '_> {
 
         ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32);
         ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64);
+
         ifn!("llvm.round.f32", fn(t_f32) -> t_f32);
         ifn!("llvm.round.f64", fn(t_f64) -> t_f64);
 
+        ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32);
+        ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64);
+
         ifn!("llvm.rint.f32", fn(t_f32) -> t_f32);
         ifn!("llvm.rint.f64", fn(t_f64) -> t_f64);
         ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32);
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
index 80fd9726f..ff2b005d7 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
@@ -5,12 +5,12 @@ use crate::llvm;
 use crate::builder::Builder;
 use crate::common::CodegenCx;
 use crate::value::Value;
+use rustc_ast::attr;
 use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive;
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_session::config::{CrateType, DebugInfo};
-
 use rustc_span::symbol::sym;
 use rustc_span::DebuggerVisualizerType;
 
@@ -87,7 +87,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, '
 
 pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool {
     let omit_gdb_pretty_printer_section =
-        cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section);
+        attr::contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section);
 
     // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create
     // ODR violations at link time, this section will not be emitted for rlibs since
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
index c1b3f34e5..21a0a60b0 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@@ -176,15 +176,14 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>(
 
     return_if_di_node_created_in_meantime!(cx, unique_type_id);
 
-    let (thin_pointer_size, thin_pointer_align) =
-        cx.size_and_align_of(cx.tcx.mk_imm_ptr(cx.tcx.types.unit));
+    let data_layout = &cx.tcx.data_layout;
     let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true);
 
     match fat_pointer_kind(cx, pointee_type) {
         None => {
             // This is a thin pointer. Create a regular pointer type and give it the correct name.
             debug_assert_eq!(
-                (thin_pointer_size, thin_pointer_align),
+                (data_layout.pointer_size, data_layout.pointer_align.abi),
                 cx.size_and_align_of(ptr_type),
                 "ptr_type={}, pointee_type={}",
                 ptr_type,
@@ -195,8 +194,8 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>(
                 llvm::LLVMRustDIBuilderCreatePointerType(
                     DIB(cx),
                     pointee_type_di_node,
-                    thin_pointer_size.bits(),
-                    thin_pointer_align.bits() as u32,
+                    data_layout.pointer_size.bits(),
+                    data_layout.pointer_align.abi.bits() as u32,
                     0, // Ignore DWARF address space.
                     ptr_type_debuginfo_name.as_ptr().cast(),
                     ptr_type_debuginfo_name.len(),
@@ -831,24 +830,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>(
     }
     .unwrap_or_default();
     let split_name = split_name.to_str().unwrap();
-
-    // FIXME(#60020):
-    //
-    //    This should actually be
-    //
-    //        let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo);
-    //
-    //    That is, we should set LLVM's emission kind to `LineTablesOnly` if
-    //    we are compiling with "limited" debuginfo. However, some of the
-    //    existing tools relied on slightly more debuginfo being generated than
-    //    would be the case with `LineTablesOnly`, and we did not want to break
-    //    these tools in a "drive-by fix", without a good idea or plan about
-    //    what limited debuginfo should exactly look like. So for now we keep
-    //    the emission kind as `FullDebug`.
-    //
-    //    See https://github.com/rust-lang/rust/issues/60020 for details.
-    let kind = DebugEmissionKind::FullDebug;
-    assert!(tcx.sess.opts.debuginfo != DebugInfo::None);
+    let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo);
 
     unsafe {
         let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile(
@@ -882,8 +864,6 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>(
         );
 
         if tcx.sess.opts.unstable_opts.profile {
-            let cu_desc_metadata =
-                llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata);
             let default_gcda_path = &output_filenames.with_extension("gcda");
             let gcda_path =
                 tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path);
@@ -891,20 +871,17 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>(
             let gcov_cu_info = [
                 path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")),
                 path_to_mdstring(debug_context.llcontext, gcda_path),
-                cu_desc_metadata,
+                unit_metadata,
             ];
-            let gcov_metadata = llvm::LLVMMDNodeInContext(
+            let gcov_metadata = llvm::LLVMMDNodeInContext2(
                 debug_context.llcontext,
                 gcov_cu_info.as_ptr(),
-                gcov_cu_info.len() as c_uint,
+                gcov_cu_info.len(),
             );
+            let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata);
 
             let llvm_gcov_ident = cstr!("llvm.gcov");
-            llvm::LLVMAddNamedMetadataOperand(
-                debug_context.llmod,
-                llvm_gcov_ident.as_ptr(),
-                gcov_metadata,
-            );
+            llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, llvm_gcov_ident.as_ptr(), val);
         }
 
         // Insert `llvm.ident` metadata on the wasm targets since that will
@@ -925,15 +902,9 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>(
         return unit_metadata;
     };
 
-    fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll Value {
+    fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll llvm::Metadata {
         let path_str = path_to_c_string(path);
-        unsafe {
-            llvm::LLVMMDStringInContext(
-                llcx,
-                path_str.as_ptr(),
-                path_str.as_bytes().len() as c_uint,
-            )
-        }
+        unsafe { llvm::LLVMMDStringInContext2(llcx, path_str.as_ptr(), path_str.as_bytes().len()) }
     }
 }
 
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
index 54e850f25..55a217f59 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
@@ -3,10 +3,10 @@ use rustc_codegen_ssa::debuginfo::{
     wants_c_like_enum_debuginfo,
 };
 use rustc_hir::def::CtorKind;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::IndexSlice;
 use rustc_middle::{
     bug,
-    mir::{Field, GeneratorLayout, GeneratorSavedLocal},
+    mir::{GeneratorLayout, GeneratorSavedLocal},
     ty::{
         self,
         layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout},
@@ -14,7 +14,9 @@ use rustc_middle::{
     },
 };
 use rustc_span::Symbol;
-use rustc_target::abi::{HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants};
+use rustc_target::abi::{
+    FieldIdx, HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants,
+};
 use std::borrow::Cow;
 
 use crate::{
@@ -272,7 +274,8 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>(
                 .map(|field_index| {
                     let field_name = if variant_def.ctor_kind() != Some(CtorKind::Fn) {
                         // Fields have names
-                        Cow::from(variant_def.fields[field_index].name.as_str())
+                        let field = &variant_def.fields[FieldIdx::from_usize(field_index)];
+                        Cow::from(field.name.as_str())
                     } else {
                         // Tuple-like
                         super::tuple_field_name(field_index)
@@ -320,7 +323,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>(
     generator_type_and_layout: TyAndLayout<'tcx>,
     generator_type_di_node: &'ll DIType,
     generator_layout: &GeneratorLayout<'tcx>,
-    state_specific_upvar_names: &IndexVec<GeneratorSavedLocal, Option<Symbol>>,
+    state_specific_upvar_names: &IndexSlice<GeneratorSavedLocal, Option<Symbol>>,
     common_upvar_names: &[String],
 ) -> &'ll DIType {
     let variant_name = GeneratorSubsts::variant_name(variant_index);
@@ -353,7 +356,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>(
             let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count())
                 .map(|field_index| {
                     let generator_saved_local = generator_layout.variant_fields[variant_index]
-                        [Field::from_usize(field_index)];
+                        [FieldIdx::from_usize(field_index)];
                     let field_name_maybe = state_specific_upvar_names[generator_saved_local];
                     let field_name = field_name_maybe
                         .as_ref()
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
index 93419d27a..978141917 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
@@ -438,6 +438,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>(
 ///         DW_TAG_structure_type            (type of variant 1)
 ///         DW_TAG_structure_type            (type of variant 2)
 ///         DW_TAG_structure_type            (type of variant 3)
+/// ```
 struct VariantMemberInfo<'a, 'll> {
     variant_index: VariantIdx,
     variant_name: Cow<'a, str>,
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index 5392534cf..d808f8c5f 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -209,8 +209,7 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
 
     fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) {
         unsafe {
-            let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc);
-            llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval);
+            llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, dbg_loc);
         }
     }
 
@@ -322,7 +321,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         let tcx = self.tcx;
 
         let def_id = instance.def_id();
-        let containing_scope = get_containing_scope(self, instance);
+        let (containing_scope, is_method) = get_containing_scope(self, instance);
         let span = tcx.def_span(def_id);
         let loc = self.lookup_debug_loc(span.lo());
         let file_metadata = file_metadata(self, &loc.file);
@@ -378,8 +377,29 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             }
         }
 
-        unsafe {
-            return llvm::LLVMRustDIBuilderCreateFunction(
+        // When we're adding a method to a type DIE, we only want a DW_AT_declaration there, because
+        // LLVM LTO can't unify type definitions when a child DIE is a full subprogram definition.
+        // When we use this `decl` below, the subprogram definition gets created at the CU level
+        // with a DW_AT_specification pointing back to the type's declaration.
+        let decl = is_method.then(|| unsafe {
+            llvm::LLVMRustDIBuilderCreateMethod(
+                DIB(self),
+                containing_scope,
+                name.as_ptr().cast(),
+                name.len(),
+                linkage_name.as_ptr().cast(),
+                linkage_name.len(),
+                file_metadata,
+                loc.line,
+                function_type_metadata,
+                flags,
+                spflags & !DISPFlags::SPFlagDefinition,
+                template_parameters,
+            )
+        });
+
+        return unsafe {
+            llvm::LLVMRustDIBuilderCreateFunction(
                 DIB(self),
                 containing_scope,
                 name.as_ptr().cast(),
@@ -394,15 +414,15 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 spflags,
                 maybe_definition_llfn,
                 template_parameters,
-                None,
-            );
-        }
+                decl,
+            )
+        };
 
         fn get_function_signature<'ll, 'tcx>(
             cx: &CodegenCx<'ll, 'tcx>,
             fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         ) -> &'ll DIArray {
-            if cx.sess().opts.debuginfo == DebugInfo::Limited {
+            if cx.sess().opts.debuginfo != DebugInfo::Full {
                 return create_DIArray(DIB(cx), &[]);
             }
 
@@ -495,14 +515,16 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             names
         }
 
+        /// Returns a scope, plus `true` if that's a type scope for "class" methods,
+        /// otherwise `false` for plain namespace scopes.
         fn get_containing_scope<'ll, 'tcx>(
             cx: &CodegenCx<'ll, 'tcx>,
             instance: Instance<'tcx>,
-        ) -> &'ll DIScope {
+        ) -> (&'ll DIScope, bool) {
             // First, let's see if this is a method within an inherent impl. Because
             // if yes, we want to make the result subroutine DIE a child of the
             // subroutine's self-type.
-            let self_type = cx.tcx.impl_of_method(instance.def_id()).and_then(|impl_def_id| {
+            if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) {
                 // If the method does *not* belong to a trait, proceed
                 if cx.tcx.trait_id_of_impl(impl_def_id).is_none() {
                     let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions(
@@ -513,39 +535,34 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
 
                     // Only "class" methods are generally understood by LLVM,
                     // so avoid methods on other types (e.g., `<*mut T>::null`).
-                    match impl_self_ty.kind() {
-                        ty::Adt(def, ..) if !def.is_box() => {
-                            // Again, only create type information if full debuginfo is enabled
-                            if cx.sess().opts.debuginfo == DebugInfo::Full
-                                && !impl_self_ty.needs_subst()
-                            {
-                                Some(type_di_node(cx, impl_self_ty))
-                            } else {
-                                Some(namespace::item_namespace(cx, def.did()))
-                            }
+                    if let ty::Adt(def, ..) = impl_self_ty.kind() && !def.is_box() {
+                        // Again, only create type information if full debuginfo is enabled
+                        if cx.sess().opts.debuginfo == DebugInfo::Full
+                            && !impl_self_ty.needs_subst()
+                        {
+                            return (type_di_node(cx, impl_self_ty), true);
+                        } else {
+                            return (namespace::item_namespace(cx, def.did()), false);
                         }
-                        _ => None,
                     }
                 } else {
                     // For trait method impls we still use the "parallel namespace"
                     // strategy
-                    None
                 }
-            });
+            }
 
-            self_type.unwrap_or_else(|| {
-                namespace::item_namespace(
-                    cx,
-                    DefId {
-                        krate: instance.def_id().krate,
-                        index: cx
-                            .tcx
-                            .def_key(instance.def_id())
-                            .parent
-                            .expect("get_containing_scope: missing parent?"),
-                    },
-                )
-            })
+            let scope = namespace::item_namespace(
+                cx,
+                DefId {
+                    krate: instance.def_id().krate,
+                    index: cx
+                        .tcx
+                        .def_key(instance.def_id())
+                        .parent
+                        .expect("get_containing_scope: missing parent?"),
+                },
+            );
+            (scope, false)
         }
     }
 
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
index 5cd0e1cb6..6bcd3e5bf 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
@@ -5,7 +5,7 @@ use super::CodegenUnitDebugContext;
 
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::layout::{HasParamEnv, LayoutOf};
-use rustc_middle::ty::{self, DefIdTree, Ty};
+use rustc_middle::ty::{self, Ty};
 use trace;
 
 use crate::common::CodegenCx;
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 39afb4af6..012e25884 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -71,6 +71,8 @@ fn get_simple_intrinsic<'ll>(
         sym::roundf32 => "llvm.round.f32",
         sym::roundf64 => "llvm.round.f64",
         sym::ptr_mask => "llvm.ptrmask",
+        sym::roundevenf32 => "llvm.roundeven.f32",
+        sym::roundevenf64 => "llvm.roundeven.f64",
         _ => return None,
     };
     Some(cx.get_intrinsic(llvm_name))
@@ -376,7 +378,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 }
             }
 
-            _ => bug!("unknown intrinsic '{}'", name),
+            _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name),
         };
 
         if !fn_abi.ret.is_ignore() {
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index c41e74c51..3f77ea77e 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -10,7 +10,6 @@
 #![feature(iter_intersperse)]
 #![feature(let_chains)]
 #![feature(never_type)]
-#![feature(once_cell)]
 #![recursion_limit = "256"]
 #![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
@@ -84,7 +83,7 @@ mod type_of;
 mod va_arg;
 mod value;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 #[derive(Clone)]
 pub struct LlvmCodegenBackend(());
@@ -361,12 +360,12 @@ impl CodegenBackend for LlvmCodegenBackend {
             .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>")
             .join(sess);
 
-        sess.time("llvm_dump_timing_file", || {
-            if sess.opts.unstable_opts.llvm_time_trace {
+        if sess.opts.unstable_opts.llvm_time_trace {
+            sess.time("llvm_dump_timing_file", || {
                 let file_name = outputs.with_extension("llvm_timings.json");
                 llvm_util::time_trace_profiler_finish(&file_name);
-            }
-        });
+            });
+        }
 
         Ok((codegen_results, work_products))
     }
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 253c2ca7c..05bbdbb74 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -641,9 +641,6 @@ pub struct Builder<'a>(InvariantOpaque<'a>);
 #[repr(C)]
 pub struct PassManager<'a>(InvariantOpaque<'a>);
 extern "C" {
-    pub type PassManagerBuilder;
-}
-extern "C" {
     pub type Pass;
 }
 extern "C" {
@@ -949,15 +946,27 @@ pub mod debuginfo {
         NoDebug,
         FullDebug,
         LineTablesOnly,
+        DebugDirectivesOnly,
     }
 
     impl DebugEmissionKind {
         pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self {
+            // We should be setting LLVM's emission kind to `LineTablesOnly` if
+            // we are compiling with "limited" debuginfo. However, some of the
+            // existing tools relied on slightly more debuginfo being generated than
+            // would be the case with `LineTablesOnly`, and we did not want to break
+            // these tools in a "drive-by fix", without a good idea or plan about
+            // what limited debuginfo should exactly look like. So for now we are
+            // instead adding a new debuginfo option "line-tables-only" so as to
+            // not break anything and to allow users to have 'limited' debug info.
+            //
+            // See https://github.com/rust-lang/rust/issues/60020 for details.
             use rustc_session::config::DebugInfo;
             match kind {
                 DebugInfo::None => DebugEmissionKind::NoDebug,
-                DebugInfo::Limited => DebugEmissionKind::LineTablesOnly,
-                DebugInfo::Full => DebugEmissionKind::FullDebug,
+                DebugInfo::LineDirectivesOnly => DebugEmissionKind::DebugDirectivesOnly,
+                DebugInfo::LineTablesOnly => DebugEmissionKind::LineTablesOnly,
+                DebugInfo::Limited | DebugInfo::Full => DebugEmissionKind::FullDebug,
             }
         }
     }
@@ -1009,7 +1018,7 @@ extern "C" {
     pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char);
 
     /// See Module::setModuleInlineAsm.
-    pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t);
+    pub fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t);
 
     /// See llvm::LLVMTypeKind::getTypeID.
     pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind;
@@ -1056,7 +1065,7 @@ extern "C" {
 
     // Operations on other types
     pub fn LLVMVoidTypeInContext(C: &Context) -> &Type;
-    pub fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type;
+    pub fn LLVMMetadataTypeInContext(C: &Context) -> &Type;
 
     // Operations on all values
     pub fn LLVMTypeOf(Val: &Value) -> &Type;
@@ -1072,9 +1081,15 @@ extern "C" {
     // Operations on constants of any type
     pub fn LLVMConstNull(Ty: &Type) -> &Value;
     pub fn LLVMGetUndef(Ty: &Type) -> &Value;
+    pub fn LLVMGetPoison(Ty: &Type) -> &Value;
 
     // Operations on metadata
+    // FIXME: deprecated, replace with LLVMMDStringInContext2
     pub fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value;
+
+    pub fn LLVMMDStringInContext2(C: &Context, Str: *const c_char, SLen: size_t) -> &Metadata;
+
+    // FIXME: deprecated, replace with LLVMMDNodeInContext2
     pub fn LLVMMDNodeInContext<'a>(
         C: &'a Context,
         Vals: *const &'a Value,
@@ -1113,6 +1128,8 @@ extern "C" {
         Packed: Bool,
     ) -> &'a Value;
 
+    // FIXME: replace with LLVMConstArray2 when bumped minimal version to llvm-17
+    // https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc
     pub fn LLVMConstArray<'a>(
         ElementTy: &'a Type,
         ConstantVals: *const &'a Value,
@@ -1252,7 +1269,7 @@ extern "C" {
     pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>);
 
     // Metadata
-    pub fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: &'a Value);
+    pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: &'a Metadata);
 
     // Terminators
     pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value;
@@ -1292,38 +1309,38 @@ extern "C" {
     pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value;
     pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value;
 
-    pub fn LLVMRustBuildCleanupPad<'a>(
+    pub fn LLVMBuildCleanupPad<'a>(
         B: &Builder<'a>,
         ParentPad: Option<&'a Value>,
-        ArgCnt: c_uint,
         Args: *const &'a Value,
+        NumArgs: c_uint,
         Name: *const c_char,
     ) -> Option<&'a Value>;
-    pub fn LLVMRustBuildCleanupRet<'a>(
+    pub fn LLVMBuildCleanupRet<'a>(
         B: &Builder<'a>,
         CleanupPad: &'a Value,
-        UnwindBB: Option<&'a BasicBlock>,
+        BB: Option<&'a BasicBlock>,
     ) -> Option<&'a Value>;
-    pub fn LLVMRustBuildCatchPad<'a>(
+    pub fn LLVMBuildCatchPad<'a>(
         B: &Builder<'a>,
         ParentPad: &'a Value,
-        ArgCnt: c_uint,
         Args: *const &'a Value,
+        NumArgs: c_uint,
         Name: *const c_char,
     ) -> Option<&'a Value>;
-    pub fn LLVMRustBuildCatchRet<'a>(
+    pub fn LLVMBuildCatchRet<'a>(
         B: &Builder<'a>,
-        Pad: &'a Value,
+        CatchPad: &'a Value,
         BB: &'a BasicBlock,
     ) -> Option<&'a Value>;
-    pub fn LLVMRustBuildCatchSwitch<'a>(
+    pub fn LLVMBuildCatchSwitch<'a>(
         Builder: &Builder<'a>,
         ParentPad: Option<&'a Value>,
-        BB: Option<&'a BasicBlock>,
+        UnwindBB: Option<&'a BasicBlock>,
         NumHandlers: c_uint,
         Name: *const c_char,
     ) -> Option<&'a Value>;
-    pub fn LLVMRustAddHandler<'a>(CatchSwitch: &'a Value, Handler: &'a BasicBlock);
+    pub fn LLVMAddHandler<'a>(CatchSwitch: &'a Value, Dest: &'a BasicBlock);
     pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value);
 
     // Add a case to the switch instruction
@@ -1617,11 +1634,12 @@ extern "C" {
         DestTy: &'a Type,
         Name: *const c_char,
     ) -> &'a Value;
-    pub fn LLVMRustBuildIntCast<'a>(
+    pub fn LLVMBuildIntCast2<'a>(
         B: &Builder<'a>,
         Val: &'a Value,
         DestTy: &'a Type,
-        IsSigned: bool,
+        IsSigned: Bool,
+        Name: *const c_char,
     ) -> &'a Value;
 
     // Comparisons
@@ -1910,7 +1928,7 @@ extern "C" {
     );
     pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool;
 
-    pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value;
+    pub fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value;
 
     pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>;
 
@@ -1968,6 +1986,21 @@ extern "C" {
         Decl: Option<&'a DIDescriptor>,
     ) -> &'a DISubprogram;
 
+    pub fn LLVMRustDIBuilderCreateMethod<'a>(
+        Builder: &DIBuilder<'a>,
+        Scope: &'a DIDescriptor,
+        Name: *const c_char,
+        NameLen: size_t,
+        LinkageName: *const c_char,
+        LinkageNameLen: size_t,
+        File: &'a DIFile,
+        LineNo: c_uint,
+        Ty: &'a DIType,
+        Flags: DIFlags,
+        SPFlags: DISPFlags,
+        TParam: &'a DIArray,
+    ) -> &'a DISubprogram;
+
     pub fn LLVMRustDIBuilderCreateBasicType<'a>(
         Builder: &DIBuilder<'a>,
         Name: *const c_char,
@@ -2259,6 +2292,7 @@ extern "C" {
         RelaxELFRelocations: bool,
         UseInitArray: bool,
         SplitDwarfFile: *const c_char,
+        ForceEmulatedTls: bool,
     ) -> Option<&'static mut TargetMachine>;
     pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine);
     pub fn LLVMRustAddLibraryInfo<'a>(
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index ba58a2e68..46692fd5e 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -424,7 +424,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str
         .filter_map(|s| {
             let enable_disable = match s.chars().next() {
                 None => return None,
-                Some(c @ '+' | c @ '-') => c,
+                Some(c @ ('+' | '-')) => c,
                 Some(_) => {
                     if diagnostics {
                         sess.emit_warning(UnknownCTargetFeaturePrefix { feature: s });
diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs
index ff111d96f..bef4647f2 100644
--- a/compiler/rustc_codegen_llvm/src/type_.rs
+++ b/compiler/rustc_codegen_llvm/src/type_.rs
@@ -53,7 +53,7 @@ impl<'ll> CodegenCx<'ll, '_> {
     }
 
     pub(crate) fn type_metadata(&self) -> &'ll Type {
-        unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) }
+        unsafe { llvm::LLVMMetadataTypeInContext(self.llcx) }
     }
 
     ///x Creates an integer type with the given number of bits, e.g., i24
diff --git a/compiler/rustc_codegen_ssa/locales/en-US.ftl b/compiler/rustc_codegen_ssa/messages.ftl
index 8fe5f8d50..243be0e1f 100644
--- a/compiler/rustc_codegen_ssa/locales/en-US.ftl
+++ b/compiler/rustc_codegen_ssa/messages.ftl
@@ -141,7 +141,7 @@ codegen_ssa_msvc_missing_linker = the msvc targets depend on the msvc linker but
 
 codegen_ssa_check_installed_visual_studio = please ensure that Visual Studio 2017 or later, or Build Tools for Visual Studio were installed with the Visual C++ option.
 
-codegen_ssa_unsufficient_vs_code_product = VS Code is a different product, and is not sufficient.
+codegen_ssa_insufficient_vs_code_product = VS Code is a different product, and is not sufficient.
 
 codegen_ssa_processing_dymutil_failed = processing debug info with `dsymutil` failed: {$status}
     .note = {$output}
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 8bb143ed3..eecfe13bb 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -349,7 +349,10 @@ fn link_rlib<'a>(
         let NativeLibKind::Static { bundle: None | Some(true), whole_archive } = lib.kind else {
             continue;
         };
-        if whole_archive == Some(true) && !codegen_results.crate_info.feature_packed_bundled_libs {
+        if whole_archive == Some(true)
+            && flavor == RlibFlavor::Normal
+            && !codegen_results.crate_info.feature_packed_bundled_libs
+        {
             sess.emit_err(errors::IncompatibleLinkingModifiers);
         }
         if flavor == RlibFlavor::Normal && let Some(filename) = lib.filename {
@@ -358,9 +361,9 @@ fn link_rlib<'a>(
             let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
             let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
             packed_bundled_libs.push(wrapper_file);
-        } else if let Some(name) = lib.name {
+        } else {
             let path =
-                find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
+                find_native_static_library(lib.name.as_str(), lib.verbatim, &lib_search_paths, sess);
             ab.add_archive(&path, Box::new(|_| false)).unwrap_or_else(|error| {
                 sess.emit_fatal(errors::AddNativeLibrary { library_path: path, error })});
         }
@@ -436,7 +439,7 @@ fn collate_raw_dylibs<'a, 'b>(
     for lib in used_libraries {
         if lib.kind == NativeLibKind::RawDylib {
             let ext = if lib.verbatim { "" } else { ".dll" };
-            let name = format!("{}{}", lib.name.expect("unnamed raw-dylib library"), ext);
+            let name = format!("{}{}", lib.name, ext);
             let imports = dylib_table.entry(name.clone()).or_default();
             for import in &lib.dll_imports {
                 if let Some(old_import) = imports.insert(import.name, import) {
@@ -923,7 +926,7 @@ fn link_natively<'a>(
             if sess.target.is_like_msvc && linker_not_found {
                 sess.emit_note(errors::MsvcMissingLinker);
                 sess.emit_note(errors::CheckInstalledVisualStudio);
-                sess.emit_note(errors::UnsufficientVSCodeProduct);
+                sess.emit_note(errors::InsufficientVSCodeProduct);
             }
             sess.abort_if_errors();
         }
@@ -1296,7 +1299,7 @@ fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) {
         .iter()
         .filter(|l| relevant_lib(sess, l))
         .filter_map(|lib| {
-            let name = lib.name?;
+            let name = lib.name;
             match lib.kind {
                 NativeLibKind::Static { bundle: Some(false), .. }
                 | NativeLibKind::Dylib { .. }
@@ -1317,6 +1320,7 @@ fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) {
                 // These are included, no need to print them
                 NativeLibKind::Static { bundle: None | Some(true), .. }
                 | NativeLibKind::LinkArg
+                | NativeLibKind::WasmImportModule
                 | NativeLibKind::RawDylib => None,
             }
         })
@@ -2275,21 +2279,18 @@ fn add_native_libs_from_crate(
 
     let mut last = (None, NativeLibKind::Unspecified, false);
     for lib in native_libs {
-        let Some(name) = lib.name else {
-            continue;
-        };
         if !relevant_lib(sess, lib) {
             continue;
         }
 
         // Skip if this library is the same as the last.
-        last = if (lib.name, lib.kind, lib.verbatim) == last {
+        last = if (Some(lib.name), lib.kind, lib.verbatim) == last {
             continue;
         } else {
-            (lib.name, lib.kind, lib.verbatim)
+            (Some(lib.name), lib.kind, lib.verbatim)
         };
 
-        let name = name.as_str();
+        let name = lib.name.as_str();
         let verbatim = lib.verbatim;
         match lib.kind {
             NativeLibKind::Static { bundle, whole_archive } => {
@@ -2303,7 +2304,7 @@ fn add_native_libs_from_crate(
                         || (whole_archive == None
                             && bundle
                             && cnum == LOCAL_CRATE
-                            && sess.opts.test);
+                            && sess.is_test_crate());
 
                     if bundle && cnum != LOCAL_CRATE {
                         if let Some(filename) = lib.filename {
@@ -2346,6 +2347,7 @@ fn add_native_libs_from_crate(
             NativeLibKind::RawDylib => {
                 // Handled separately in `linker_with_args`.
             }
+            NativeLibKind::WasmImportModule => {}
             NativeLibKind::LinkArg => {
                 if link_static {
                     cmd.arg(name);
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index 52c01b423..65dfc325a 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -133,6 +133,9 @@ pub fn get_linker<'a>(
         LinkerFlavor::Unix(Cc::No) if sess.target.os == "l4re" => {
             Box::new(L4Bender::new(cmd, sess)) as Box<dyn Linker>
         }
+        LinkerFlavor::Unix(Cc::No) if sess.target.os == "aix" => {
+            Box::new(AixLinker::new(cmd, sess)) as Box<dyn Linker>
+        }
         LinkerFlavor::WasmLld(Cc::No) => Box::new(WasmLd::new(cmd, sess)) as Box<dyn Linker>,
         LinkerFlavor::Gnu(cc, _)
         | LinkerFlavor::Darwin(cc, _)
@@ -720,6 +723,7 @@ impl<'a> Linker for GccLinker<'a> {
                 let mut arg = OsString::from("--version-script=");
                 arg.push(path);
                 self.linker_arg(arg);
+                self.linker_arg("--no-undefined-version");
             }
         }
     }
@@ -1117,9 +1121,12 @@ impl<'a> Linker for EmLinker<'a> {
 
     fn debuginfo(&mut self, _strip: Strip, _: &[PathBuf]) {
         // Preserve names or generate source maps depending on debug info
+        // For more information see https://emscripten.org/docs/tools_reference/emcc.html#emcc-g
         self.cmd.arg(match self.sess.opts.debuginfo {
             DebugInfo::None => "-g0",
-            DebugInfo::Limited => "--profiling-funcs",
+            DebugInfo::Limited | DebugInfo::LineTablesOnly | DebugInfo::LineDirectivesOnly => {
+                "--profiling-funcs"
+            }
             DebugInfo::Full => "-g",
         });
     }
@@ -1473,6 +1480,177 @@ impl<'a> L4Bender<'a> {
     }
 }
 
+/// Linker for AIX.
+pub struct AixLinker<'a> {
+    cmd: Command,
+    sess: &'a Session,
+    hinted_static: bool,
+}
+
+impl<'a> AixLinker<'a> {
+    pub fn new(cmd: Command, sess: &'a Session) -> AixLinker<'a> {
+        AixLinker { cmd: cmd, sess: sess, hinted_static: false }
+    }
+
+    fn hint_static(&mut self) {
+        if !self.hinted_static {
+            self.cmd.arg("-bstatic");
+            self.hinted_static = true;
+        }
+    }
+
+    fn hint_dynamic(&mut self) {
+        if self.hinted_static {
+            self.cmd.arg("-bdynamic");
+            self.hinted_static = false;
+        }
+    }
+
+    fn build_dylib(&mut self, _out_filename: &Path) {
+        self.cmd.arg("-bM:SRE");
+        self.cmd.arg("-bnoentry");
+        // FIXME: Use CreateExportList utility to create export list
+        // and remove -bexpfull.
+        self.cmd.arg("-bexpfull");
+    }
+}
+
+impl<'a> Linker for AixLinker<'a> {
+    fn link_dylib(&mut self, lib: &str, _verbatim: bool, _as_needed: bool) {
+        self.hint_dynamic();
+        self.cmd.arg(format!("-l{}", lib));
+    }
+
+    fn link_staticlib(&mut self, lib: &str, _verbatim: bool) {
+        self.hint_static();
+        self.cmd.arg(format!("-l{}", lib));
+    }
+
+    fn link_rlib(&mut self, lib: &Path) {
+        self.hint_static();
+        self.cmd.arg(lib);
+    }
+
+    fn include_path(&mut self, path: &Path) {
+        self.cmd.arg("-L").arg(path);
+    }
+
+    fn framework_path(&mut self, _: &Path) {
+        bug!("frameworks are not supported on AIX");
+    }
+
+    fn output_filename(&mut self, path: &Path) {
+        self.cmd.arg("-o").arg(path);
+    }
+
+    fn add_object(&mut self, path: &Path) {
+        self.cmd.arg(path);
+    }
+
+    fn full_relro(&mut self) {}
+
+    fn partial_relro(&mut self) {}
+
+    fn no_relro(&mut self) {}
+
+    fn cmd(&mut self) -> &mut Command {
+        &mut self.cmd
+    }
+
+    fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path) {
+        match output_kind {
+            LinkOutputKind::DynamicDylib => {
+                self.hint_dynamic();
+                self.build_dylib(out_filename);
+            }
+            LinkOutputKind::StaticDylib => {
+                self.hint_static();
+                self.build_dylib(out_filename);
+            }
+            _ => {}
+        }
+    }
+
+    fn link_rust_dylib(&mut self, lib: &str, _: &Path) {
+        self.hint_dynamic();
+        self.cmd.arg(format!("-l{}", lib));
+    }
+
+    fn link_framework(&mut self, _framework: &str, _as_needed: bool) {
+        bug!("frameworks not supported on AIX");
+    }
+
+    fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, search_path: &[PathBuf]) {
+        self.hint_static();
+        let lib = find_native_static_library(lib, verbatim, search_path, &self.sess);
+        self.cmd.arg(format!("-bkeepfile:{}", lib.to_str().unwrap()));
+    }
+
+    fn link_whole_rlib(&mut self, lib: &Path) {
+        self.hint_static();
+        self.cmd.arg(format!("-bkeepfile:{}", lib.to_str().unwrap()));
+    }
+
+    fn gc_sections(&mut self, _keep_metadata: bool) {
+        self.cmd.arg("-bgc");
+    }
+
+    fn no_gc_sections(&mut self) {
+        self.cmd.arg("-bnogc");
+    }
+
+    fn optimize(&mut self) {}
+
+    fn pgo_gen(&mut self) {}
+
+    fn control_flow_guard(&mut self) {}
+
+    fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) {
+        match strip {
+            Strip::None => {}
+            // FIXME: -s strips the symbol table, line number information
+            // and relocation information.
+            Strip::Debuginfo | Strip::Symbols => {
+                self.cmd.arg("-s");
+            }
+        }
+    }
+
+    fn no_crt_objects(&mut self) {}
+
+    fn no_default_libraries(&mut self) {}
+
+    fn export_symbols(&mut self, tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) {
+        let path = tmpdir.join("list.exp");
+        let res: io::Result<()> = try {
+            let mut f = BufWriter::new(File::create(&path)?);
+            // FIXME: use llvm-nm to generate export list.
+            for symbol in symbols {
+                debug!("  _{}", symbol);
+                writeln!(f, "  {}", symbol)?;
+            }
+        };
+        if let Err(e) = res {
+            self.sess.fatal(&format!("failed to write export file: {}", e));
+        }
+        self.cmd.arg(format!("-bE:{}", path.to_str().unwrap()));
+    }
+
+    fn subsystem(&mut self, _subsystem: &str) {}
+
+    fn reset_per_library_state(&mut self) {
+        self.hint_dynamic();
+    }
+
+    fn linker_plugin_lto(&mut self) {}
+
+    fn add_eh_frame_header(&mut self) {}
+
+    fn add_no_exec(&mut self) {}
+
+    fn add_as_needed(&mut self) {}
+}
+
 fn for_each_exported_symbols_include_dep<'tcx>(
     tcx: TyCtxt<'tcx>,
     crate_type: CrateType,
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index 019ec0758..d5d843702 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -13,8 +13,7 @@ use object::{
 use snap::write::FrameEncoder;
 
 use rustc_data_structures::memmap::Mmap;
-use rustc_data_structures::owning_ref::OwningRef;
-use rustc_data_structures::rustc_erase_owner;
+use rustc_data_structures::owned_slice::try_slice_owned;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_metadata::fs::METADATA_FILENAME;
 use rustc_metadata::EncodedMetadata;
@@ -42,10 +41,10 @@ fn load_metadata_with(
 ) -> Result<MetadataRef, String> {
     let file =
         File::open(path).map_err(|e| format!("failed to open file '{}': {}", path.display(), e))?;
-    let data = unsafe { Mmap::map(file) }
-        .map_err(|e| format!("failed to mmap file '{}': {}", path.display(), e))?;
-    let metadata = OwningRef::new(data).try_map(f)?;
-    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
+
+    unsafe { Mmap::map(file) }
+        .map_err(|e| format!("failed to mmap file '{}': {}", path.display(), e))
+        .and_then(|mmap| try_slice_owned(mmap, |mmap| f(mmap)))
 }
 
 impl MetadataLoader for DefaultMetadataLoader {
@@ -128,6 +127,7 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
         "msp430" => Architecture::Msp430,
         "hexagon" => Architecture::Hexagon,
         "bpf" => Architecture::Bpf,
+        "loongarch64" => Architecture::LoongArch64,
         // Unsupported architecture.
         _ => return None,
     };
@@ -191,6 +191,10 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
             }
             e_flags
         }
+        Architecture::LoongArch64 => {
+            // Source: https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html#_e_flags_identifies_abi_type_and_version
+            elf::EF_LARCH_OBJABI_V1 | elf::EF_LARCH_ABI_DOUBLE_FLOAT
+        }
         _ => 0,
     };
     // adapted from LLVM's `MCELFObjectTargetWriter::getOSABI`
@@ -306,7 +310,13 @@ pub fn create_compressed_metadata_file(
     symbol_name: &str,
 ) -> Vec<u8> {
     let mut compressed = rustc_metadata::METADATA_HEADER.to_vec();
+    // Our length will be backfilled once we're done writing
+    compressed.write_all(&[0; 4]).unwrap();
     FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap();
+    let meta_len = rustc_metadata::METADATA_HEADER.len();
+    let data_len = (compressed.len() - meta_len - 4) as u32;
+    compressed[meta_len..meta_len + 4].copy_from_slice(&data_len.to_be_bytes());
+
     let Some(mut file) = create_object_file(sess) else {
         return compressed.to_vec();
     };
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index 067a3e167..d0fd3cd76 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -1,3 +1,5 @@
+use crate::base::allocator_kind_for_codegen;
+
 use std::collections::hash_map::Entry::*;
 
 use rustc_ast::expand::allocator::ALLOCATOR_METHODS;
@@ -8,10 +10,11 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::middle::exported_symbols::{
     metadata_symbol_name, ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel,
 };
+use rustc_middle::query::LocalCrate;
 use rustc_middle::ty::query::{ExternProviders, Providers};
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
 use rustc_middle::ty::Instance;
-use rustc_middle::ty::{self, DefIdTree, SymbolName, TyCtxt};
+use rustc_middle::ty::{self, SymbolName, TyCtxt};
 use rustc_session::config::{CrateType, OomStrategy};
 use rustc_target::spec::SanitizerSet;
 
@@ -39,9 +42,7 @@ pub fn crates_export_threshold(crate_types: &[CrateType]) -> SymbolExportLevel {
     }
 }
 
-fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<SymbolExportInfo> {
-    assert_eq!(cnum, LOCAL_CRATE);
-
+fn reachable_non_generics_provider(tcx: TyCtxt<'_>, _: LocalCrate) -> DefIdMap<SymbolExportInfo> {
     if !tcx.sess.opts.output_types.should_codegen() {
         return Default::default();
     }
@@ -58,7 +59,7 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
 
     let mut reachable_non_generics: DefIdMap<_> = tcx
         .reachable_set(())
-        .iter()
+        .items()
         .filter_map(|&def_id| {
             // We want to ignore some FFI functions that are not exposed from
             // this crate. Reachable FFI functions can be lumped into two
@@ -136,7 +137,7 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
             };
             (def_id.to_def_id(), info)
         })
-        .collect();
+        .into();
 
     if let Some(id) = tcx.proc_macro_decls_static(()) {
         reachable_non_generics.insert(
@@ -152,10 +153,10 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
     reachable_non_generics
 }
 
-fn is_reachable_non_generic_provider_local(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
+fn is_reachable_non_generic_provider_local(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     let export_threshold = threshold(tcx);
 
-    if let Some(&info) = tcx.reachable_non_generics(def_id.krate).get(&def_id) {
+    if let Some(&info) = tcx.reachable_non_generics(LOCAL_CRATE).get(&def_id.to_def_id()) {
         info.level.is_below_threshold(export_threshold)
     } else {
         false
@@ -168,24 +169,37 @@ fn is_reachable_non_generic_provider_extern(tcx: TyCtxt<'_>, def_id: DefId) -> b
 
 fn exported_symbols_provider_local(
     tcx: TyCtxt<'_>,
-    cnum: CrateNum,
+    _: LocalCrate,
 ) -> &[(ExportedSymbol<'_>, SymbolExportInfo)] {
-    assert_eq!(cnum, LOCAL_CRATE);
-
     if !tcx.sess.opts.output_types.should_codegen() {
         return &[];
     }
 
     // FIXME: Sorting this is unnecessary since we are sorting later anyway.
     //        Can we skip the later sorting?
-    let mut symbols: Vec<_> = tcx.with_stable_hashing_context(|hcx| {
-        tcx.reachable_non_generics(LOCAL_CRATE)
-            .to_sorted(&hcx, true)
-            .into_iter()
-            .map(|(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info))
-            .collect()
+    let sorted = tcx.with_stable_hashing_context(|hcx| {
+        tcx.reachable_non_generics(LOCAL_CRATE).to_sorted(&hcx, true)
     });
 
+    let mut symbols: Vec<_> =
+        sorted.iter().map(|(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)).collect();
+
+    // Export TLS shims
+    if !tcx.sess.target.dll_tls_export {
+        symbols.extend(sorted.iter().filter_map(|(&def_id, &info)| {
+            tcx.needs_thread_local_shim(def_id).then(|| {
+                (
+                    ExportedSymbol::ThreadLocalShim(def_id),
+                    SymbolExportInfo {
+                        level: info.level,
+                        kind: SymbolExportKind::Text,
+                        used: info.used,
+                    },
+                )
+            })
+        }))
+    }
+
     if tcx.entry_fn(()).is_some() {
         let exported_symbol =
             ExportedSymbol::NoDefId(SymbolName::new(tcx, tcx.sess.target.entry_name.as_ref()));
@@ -200,7 +214,8 @@ fn exported_symbols_provider_local(
         ));
     }
 
-    if tcx.allocator_kind(()).is_some() {
+    // Mark allocator shim symbols as exported only if they were generated.
+    if allocator_kind_for_codegen(tcx).is_some() {
         for symbol_name in ALLOCATOR_METHODS
             .iter()
             .map(|method| format!("__rust_{}", method.name))
@@ -380,7 +395,9 @@ fn upstream_monomorphizations_provider(
                         continue;
                     }
                 }
-                ExportedSymbol::NonGeneric(..) | ExportedSymbol::NoDefId(..) => {
+                ExportedSymbol::NonGeneric(..)
+                | ExportedSymbol::ThreadLocalShim(..)
+                | ExportedSymbol::NoDefId(..) => {
                     // These are no monomorphizations
                     continue;
                 }
@@ -500,6 +517,16 @@ pub fn symbol_name_for_instance_in_crate<'tcx>(
                 instantiating_crate,
             )
         }
+        ExportedSymbol::ThreadLocalShim(def_id) => {
+            rustc_symbol_mangling::symbol_name_for_instance_in_crate(
+                tcx,
+                ty::Instance {
+                    def: ty::InstanceDef::ThreadLocalShim(def_id),
+                    substs: ty::InternalSubsts::empty(),
+                },
+                instantiating_crate,
+            )
+        }
         ExportedSymbol::DropGlue(ty) => rustc_symbol_mangling::symbol_name_for_instance_in_crate(
             tcx,
             Instance::resolve_drop_in_place(tcx, ty),
@@ -548,6 +575,8 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
         ExportedSymbol::DropGlue(..) => None,
         // NoDefId always follow the target's default symbol decoration scheme.
         ExportedSymbol::NoDefId(..) => None,
+        // ThreadLocalShim always follow the target's default symbol decoration scheme.
+        ExportedSymbol::ThreadLocalShim(..) => None,
     };
 
     let (conv, args) = instance
@@ -592,7 +621,7 @@ fn wasm_import_module_map(tcx: TyCtxt<'_>, cnum: CrateNum) -> FxHashMap<DefId, S
 
     let mut ret = FxHashMap::default();
     for (def_id, lib) in tcx.foreign_modules(cnum).iter() {
-        let module = def_id_to_native_lib.get(&def_id).and_then(|s| s.wasm_import_module);
+        let module = def_id_to_native_lib.get(&def_id).and_then(|s| s.wasm_import_module());
         let Some(module) = module else { continue };
         ret.extend(lib.foreign_items.iter().map(|id| {
             assert_eq!(id.krate, cnum);
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 8508ab875..2dda4cd16 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -8,6 +8,7 @@ use crate::{
     CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
 };
 use jobserver::{Acquired, Client};
+use rustc_ast::attr;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::profiling::SelfProfilerRef;
@@ -447,8 +448,8 @@ pub fn start_async_codegen<B: ExtraBackendMethods>(
     let sess = tcx.sess;
 
     let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
-    let no_builtins = tcx.sess.contains_name(crate_attrs, sym::no_builtins);
-    let is_compiler_builtins = tcx.sess.contains_name(crate_attrs, sym::compiler_builtins);
+    let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
+    let is_compiler_builtins = attr::contains_name(crate_attrs, sym::compiler_builtins);
 
     let crate_info = CrateInfo::new(tcx, target_cpu);
 
@@ -1451,8 +1452,8 @@ fn start_executing_work<B: ExtraBackendMethods>(
                         Err(e) => {
                             let msg = &format!("failed to acquire jobserver token: {}", e);
                             shared_emitter.fatal(msg);
-                            // Exit the coordinator thread
-                            panic!("{}", msg)
+                            codegen_done = true;
+                            codegen_aborted = true;
                         }
                     }
                 }
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 73179249b..c5ca7936a 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -13,6 +13,7 @@ use crate::mir::place::PlaceRef;
 use crate::traits::*;
 use crate::{CachedModuleCodegen, CompiledModule, CrateInfo, MemFlags, ModuleCodegen, ModuleKind};
 
+use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_attr as attr;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
@@ -23,7 +24,6 @@ use rustc_data_structures::sync::ParallelIterator;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_hir::lang_items::LangItem;
-use rustc_index::vec::Idx;
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::middle::exported_symbols;
@@ -39,7 +39,7 @@ use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::Symbol;
 use rustc_span::{DebuggerVisualizerFile, DebuggerVisualizerType};
-use rustc_target::abi::{Align, VariantIdx};
+use rustc_target::abi::{Align, FIRST_VARIANT};
 
 use std::collections::BTreeSet;
 use std::time::{Duration, Instant};
@@ -306,9 +306,9 @@ pub fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {
             assert_eq!(def_a, def_b);
 
-            for i in 0..def_a.variant(VariantIdx::new(0)).fields.len() {
-                let src_f = src.project_field(bx, i);
-                let dst_f = dst.project_field(bx, i);
+            for i in def_a.variant(FIRST_VARIANT).fields.indices() {
+                let src_f = src.project_field(bx, i.as_usize());
+                let dst_f = dst.project_field(bx, i.as_usize());
 
                 if dst_f.layout.is_zst() {
                     continue;
@@ -545,6 +545,23 @@ pub fn collect_debugger_visualizers_transitive(
         .collect::<BTreeSet<_>>()
 }
 
+/// Decide allocator kind to codegen. If `Some(_)` this will be the same as
+/// `tcx.allocator_kind`, but it may be `None` in more cases (e.g. if using
+/// allocator definitions from a dylib dependency).
+pub fn allocator_kind_for_codegen(tcx: TyCtxt<'_>) -> Option<AllocatorKind> {
+    // If the crate doesn't have an `allocator_kind` set then there's definitely
+    // no shim to generate. Otherwise we also check our dependency graph for all
+    // our output crate types. If anything there looks like its a `Dynamic`
+    // linkage, then it's already got an allocator shim and we'll be using that
+    // one instead. If nothing exists then it's our job to generate the
+    // allocator!
+    let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| {
+        use rustc_middle::middle::dependency_format::Linkage;
+        list.iter().any(|&linkage| linkage == Linkage::Dynamic)
+    });
+    if any_dynamic_crate { None } else { tcx.allocator_kind(()) }
+}
+
 pub fn codegen_crate<B: ExtraBackendMethods>(
     backend: B,
     tcx: TyCtxt<'_>,
@@ -615,20 +632,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
     );
 
     // Codegen an allocator shim, if necessary.
-    //
-    // If the crate doesn't have an `allocator_kind` set then there's definitely
-    // no shim to generate. Otherwise we also check our dependency graph for all
-    // our output crate types. If anything there looks like its a `Dynamic`
-    // linkage, then it's already got an allocator shim and we'll be using that
-    // one instead. If nothing exists then it's our job to generate the
-    // allocator!
-    let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| {
-        use rustc_middle::middle::dependency_format::Linkage;
-        list.iter().any(|&linkage| linkage == Linkage::Dynamic)
-    });
-    let allocator_module = if any_dynamic_crate {
-        None
-    } else if let Some(kind) = tcx.allocator_kind(()) {
+    if let Some(kind) = allocator_kind_for_codegen(tcx) {
         let llmod_id =
             cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string();
         let module_llvm = tcx.sess.time("write_allocator_module", || {
@@ -642,13 +646,10 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
             )
         });
 
-        Some(ModuleCodegen { name: llmod_id, module_llvm, kind: ModuleKind::Allocator })
-    } else {
-        None
-    };
-
-    if let Some(allocator_module) = allocator_module {
-        ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, allocator_module);
+        ongoing_codegen.submit_pre_codegened_module_to_llvm(
+            tcx,
+            ModuleCodegen { name: llmod_id, module_llvm, kind: ModuleKind::Allocator },
+        );
     }
 
     // For better throughput during parallel processing by LLVM, we used to sort
@@ -784,6 +785,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
             total_codegen_time,
             start_rss.unwrap(),
             end_rss,
+            tcx.sess.opts.unstable_opts.time_passes_format,
         );
     }
 
@@ -807,7 +809,7 @@ impl CrateInfo {
             .collect();
         let local_crate_name = tcx.crate_name(LOCAL_CRATE);
         let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
-        let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
+        let subsystem = attr::first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
         let windows_subsystem = subsystem.map(|subsystem| {
             if subsystem != sym::windows && subsystem != sym::console {
                 tcx.sess.emit_fatal(errors::InvalidWindowsSubsystem { subsystem });
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 7d5c00486..8542bab68 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -8,8 +8,9 @@ use rustc_hir::{lang_items, weak_lang_items::WEAK_LANG_ITEMS, LangItem};
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
 use rustc_middle::mir::mono::Linkage;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self as ty, DefIdTree, TyCtxt};
+use rustc_middle::ty::{self as ty, TyCtxt};
 use rustc_session::{lint, parse::feature_err};
+use rustc_span::symbol::Ident;
 use rustc_span::{sym, Span};
 use rustc_target::spec::{abi, SanitizerSet};
 
@@ -43,7 +44,7 @@ fn linkage_by_name(tcx: TyCtxt<'_>, def_id: LocalDefId, name: &str) -> Linkage {
     }
 }
 
-fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
+fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
     if cfg!(debug_assertions) {
         let def_kind = tcx.def_kind(did);
         assert!(
@@ -52,7 +53,6 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
         );
     }
 
-    let did = did.expect_local();
     let attrs = tcx.hir().attrs(tcx.hir().local_def_id_to_hir_id(did));
     let mut codegen_fn_attrs = CodegenFnAttrs::new();
     if tcx.should_inherit_track_caller(did) {
@@ -61,351 +61,368 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
 
     let supported_target_features = tcx.supported_target_features(LOCAL_CRATE);
 
-    // In some cases, attribute are only valid on functions, but it's the `check_attr`
-    // pass that check that they aren't used anywhere else, rather this module.
-    // In these cases, we bail from performing further checks that are only meaningful for
-    // functions (such as calling `fn_sig`, which ICEs if given a non-function). We also
-    // report a delayed bug, just in case `check_attr` isn't doing its job.
-    let validate_fn_only_attr = |attr_sp| -> bool {
-        let def_kind = tcx.def_kind(did);
-        if let DefKind::Fn | DefKind::AssocFn | DefKind::Variant | DefKind::Ctor(..) = def_kind {
-            true
-        } else {
-            tcx.sess.delay_span_bug(attr_sp, "this attribute can only be applied to functions");
-            false
-        }
-    };
-
     let mut inline_span = None;
     let mut link_ordinal_span = None;
     let mut no_sanitize_span = None;
+
     for attr in attrs.iter() {
-        if attr.has_name(sym::cold) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::COLD;
-        } else if attr.has_name(sym::rustc_allocator) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR;
-        } else if attr.has_name(sym::ffi_returns_twice) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_RETURNS_TWICE;
-        } else if attr.has_name(sym::ffi_pure) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_PURE;
-        } else if attr.has_name(sym::ffi_const) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_CONST;
-        } else if attr.has_name(sym::rustc_nounwind) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND;
-        } else if attr.has_name(sym::rustc_reallocator) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::REALLOCATOR;
-        } else if attr.has_name(sym::rustc_deallocator) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::DEALLOCATOR;
-        } else if attr.has_name(sym::rustc_allocator_zeroed) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED;
-        } else if attr.has_name(sym::naked) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::NAKED;
-        } else if attr.has_name(sym::no_mangle) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE;
-        } else if attr.has_name(sym::no_coverage) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_COVERAGE;
-        } else if attr.has_name(sym::rustc_std_internal_symbol) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL;
-        } else if attr.has_name(sym::used) {
-            let inner = attr.meta_item_list();
-            match inner.as_deref() {
-                Some([item]) if item.has_name(sym::linker) => {
-                    if !tcx.features().used_with_arg {
-                        feature_err(
-                            &tcx.sess.parse_sess,
-                            sym::used_with_arg,
-                            attr.span,
-                            "`#[used(linker)]` is currently unstable",
-                        )
-                        .emit();
-                    }
-                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED_LINKER;
-                }
-                Some([item]) if item.has_name(sym::compiler) => {
-                    if !tcx.features().used_with_arg {
-                        feature_err(
-                            &tcx.sess.parse_sess,
-                            sym::used_with_arg,
+        // In some cases, attribute are only valid on functions, but it's the `check_attr`
+        // pass that check that they aren't used anywhere else, rather this module.
+        // In these cases, we bail from performing further checks that are only meaningful for
+        // functions (such as calling `fn_sig`, which ICEs if given a non-function). We also
+        // report a delayed bug, just in case `check_attr` isn't doing its job.
+        let fn_sig = || {
+            use DefKind::*;
+
+            let def_kind = tcx.def_kind(did);
+            if let Fn | AssocFn | Variant | Ctor(..) = def_kind {
+                Some(tcx.fn_sig(did))
+            } else {
+                tcx.sess
+                    .delay_span_bug(attr.span, "this attribute can only be applied to functions");
+                None
+            }
+        };
+
+        let Some(Ident { name, .. }) = attr.ident() else {
+            continue;
+        };
+
+        match name {
+            sym::cold => codegen_fn_attrs.flags |= CodegenFnAttrFlags::COLD,
+            sym::rustc_allocator => codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR,
+            sym::ffi_returns_twice => {
+                codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_RETURNS_TWICE
+            }
+            sym::ffi_pure => codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_PURE,
+            sym::ffi_const => codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_CONST,
+            sym::rustc_nounwind => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND,
+            sym::rustc_reallocator => codegen_fn_attrs.flags |= CodegenFnAttrFlags::REALLOCATOR,
+            sym::rustc_deallocator => codegen_fn_attrs.flags |= CodegenFnAttrFlags::DEALLOCATOR,
+            sym::rustc_allocator_zeroed => {
+                codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED
+            }
+            sym::naked => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NAKED,
+            sym::no_mangle => {
+                if tcx.opt_item_name(did.to_def_id()).is_some() {
+                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE
+                } else {
+                    tcx.sess
+                        .struct_span_err(
                             attr.span,
-                            "`#[used(compiler)]` is currently unstable",
+                            format!(
+                                "`#[no_mangle]` cannot be used on {} {} as it has no name",
+                                tcx.def_descr_article(did.to_def_id()),
+                                tcx.def_descr(did.to_def_id()),
+                            ),
                         )
                         .emit();
-                    }
-                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED;
                 }
-                Some(_) => {
-                    tcx.sess.emit_err(ExpectedUsedSymbol { span: attr.span });
-                }
-                None => {
-                    // Unfortunately, unconditionally using `llvm.used` causes
-                    // issues in handling `.init_array` with the gold linker,
-                    // but using `llvm.compiler.used` caused a nontrival amount
-                    // of unintentional ecosystem breakage -- particularly on
-                    // Mach-O targets.
-                    //
-                    // As a result, we emit `llvm.compiler.used` only on ELF
-                    // targets. This is somewhat ad-hoc, but actually follows
-                    // our pre-LLVM 13 behavior (prior to the ecosystem
-                    // breakage), and seems to match `clang`'s behavior as well
-                    // (both before and after LLVM 13), possibly because they
-                    // have similar compatibility concerns to us. See
-                    // https://github.com/rust-lang/rust/issues/47384#issuecomment-1019080146
-                    // and following comments for some discussion of this, as
-                    // well as the comments in `rustc_codegen_llvm` where these
-                    // flags are handled.
-                    //
-                    // Anyway, to be clear: this is still up in the air
-                    // somewhat, and is subject to change in the future (which
-                    // is a good thing, because this would ideally be a bit
-                    // more firmed up).
-                    let is_like_elf = !(tcx.sess.target.is_like_osx
-                        || tcx.sess.target.is_like_windows
-                        || tcx.sess.target.is_like_wasm);
-                    codegen_fn_attrs.flags |= if is_like_elf {
-                        CodegenFnAttrFlags::USED
-                    } else {
-                        CodegenFnAttrFlags::USED_LINKER
-                    };
-                }
-            }
-        } else if attr.has_name(sym::cmse_nonsecure_entry) {
-            if validate_fn_only_attr(attr.span)
-                && !matches!(tcx.fn_sig(did).skip_binder().abi(), abi::Abi::C { .. })
-            {
-                struct_span_err!(
-                    tcx.sess,
-                    attr.span,
-                    E0776,
-                    "`#[cmse_nonsecure_entry]` requires C ABI"
-                )
-                .emit();
             }
-            if !tcx.sess.target.llvm_target.contains("thumbv8m") {
-                struct_span_err!(tcx.sess, attr.span, E0775, "`#[cmse_nonsecure_entry]` is only valid for targets with the TrustZone-M extension")
-                    .emit();
-            }
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY;
-        } else if attr.has_name(sym::thread_local) {
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL;
-        } else if attr.has_name(sym::track_caller) {
-            if !tcx.is_closure(did.to_def_id())
-                && validate_fn_only_attr(attr.span)
-                && tcx.fn_sig(did).skip_binder().abi() != abi::Abi::Rust
-            {
-                struct_span_err!(tcx.sess, attr.span, E0737, "`#[track_caller]` requires Rust ABI")
-                    .emit();
+            sym::no_coverage => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_COVERAGE,
+            sym::rustc_std_internal_symbol => {
+                codegen_fn_attrs.flags |= CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL
             }
-            if tcx.is_closure(did.to_def_id()) && !tcx.features().closure_track_caller {
-                feature_err(
-                    &tcx.sess.parse_sess,
-                    sym::closure_track_caller,
-                    attr.span,
-                    "`#[track_caller]` on closures is currently unstable",
-                )
-                .emit();
+            sym::used => {
+                let inner = attr.meta_item_list();
+                match inner.as_deref() {
+                    Some([item]) if item.has_name(sym::linker) => {
+                        if !tcx.features().used_with_arg {
+                            feature_err(
+                                &tcx.sess.parse_sess,
+                                sym::used_with_arg,
+                                attr.span,
+                                "`#[used(linker)]` is currently unstable",
+                            )
+                            .emit();
+                        }
+                        codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED_LINKER;
+                    }
+                    Some([item]) if item.has_name(sym::compiler) => {
+                        if !tcx.features().used_with_arg {
+                            feature_err(
+                                &tcx.sess.parse_sess,
+                                sym::used_with_arg,
+                                attr.span,
+                                "`#[used(compiler)]` is currently unstable",
+                            )
+                            .emit();
+                        }
+                        codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED;
+                    }
+                    Some(_) => {
+                        tcx.sess.emit_err(ExpectedUsedSymbol { span: attr.span });
+                    }
+                    None => {
+                        // Unfortunately, unconditionally using `llvm.used` causes
+                        // issues in handling `.init_array` with the gold linker,
+                        // but using `llvm.compiler.used` caused a nontrival amount
+                        // of unintentional ecosystem breakage -- particularly on
+                        // Mach-O targets.
+                        //
+                        // As a result, we emit `llvm.compiler.used` only on ELF
+                        // targets. This is somewhat ad-hoc, but actually follows
+                        // our pre-LLVM 13 behavior (prior to the ecosystem
+                        // breakage), and seems to match `clang`'s behavior as well
+                        // (both before and after LLVM 13), possibly because they
+                        // have similar compatibility concerns to us. See
+                        // https://github.com/rust-lang/rust/issues/47384#issuecomment-1019080146
+                        // and following comments for some discussion of this, as
+                        // well as the comments in `rustc_codegen_llvm` where these
+                        // flags are handled.
+                        //
+                        // Anyway, to be clear: this is still up in the air
+                        // somewhat, and is subject to change in the future (which
+                        // is a good thing, because this would ideally be a bit
+                        // more firmed up).
+                        let is_like_elf = !(tcx.sess.target.is_like_osx
+                            || tcx.sess.target.is_like_windows
+                            || tcx.sess.target.is_like_wasm);
+                        codegen_fn_attrs.flags |= if is_like_elf {
+                            CodegenFnAttrFlags::USED
+                        } else {
+                            CodegenFnAttrFlags::USED_LINKER
+                        };
+                    }
+                }
             }
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::TRACK_CALLER;
-        } else if attr.has_name(sym::export_name) {
-            if let Some(s) = attr.value_str() {
-                if s.as_str().contains('\0') {
-                    // `#[export_name = ...]` will be converted to a null-terminated string,
-                    // so it may not contain any null characters.
+            sym::cmse_nonsecure_entry => {
+                if let Some(fn_sig) = fn_sig()
+                    && !matches!(fn_sig.skip_binder().abi(), abi::Abi::C { .. })
+                {
                     struct_span_err!(
                         tcx.sess,
                         attr.span,
-                        E0648,
-                        "`export_name` may not contain null characters"
+                        E0776,
+                        "`#[cmse_nonsecure_entry]` requires C ABI"
                     )
                     .emit();
                 }
-                codegen_fn_attrs.export_name = Some(s);
+                if !tcx.sess.target.llvm_target.contains("thumbv8m") {
+                    struct_span_err!(tcx.sess, attr.span, E0775, "`#[cmse_nonsecure_entry]` is only valid for targets with the TrustZone-M extension")
+                    .emit();
+                }
+                codegen_fn_attrs.flags |= CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY
             }
-        } else if attr.has_name(sym::target_feature) {
-            if !tcx.is_closure(did.to_def_id())
-                && tcx.fn_sig(did).skip_binder().unsafety() == hir::Unsafety::Normal
-            {
-                if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc {
-                    // The `#[target_feature]` attribute is allowed on
-                    // WebAssembly targets on all functions, including safe
-                    // ones. Other targets require that `#[target_feature]` is
-                    // only applied to unsafe functions (pending the
-                    // `target_feature_11` feature) because on most targets
-                    // execution of instructions that are not supported is
-                    // considered undefined behavior. For WebAssembly which is a
-                    // 100% safe target at execution time it's not possible to
-                    // execute undefined instructions, and even if a future
-                    // feature was added in some form for this it would be a
-                    // deterministic trap. There is no undefined behavior when
-                    // executing WebAssembly so `#[target_feature]` is allowed
-                    // on safe functions (but again, only for WebAssembly)
-                    //
-                    // Note that this is also allowed if `actually_rustdoc` so
-                    // if a target is documenting some wasm-specific code then
-                    // it's not spuriously denied.
-                } else if !tcx.features().target_feature_11 {
-                    let mut err = feature_err(
+            sym::thread_local => codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL,
+            sym::track_caller => {
+                if !tcx.is_closure(did.to_def_id())
+                    && let Some(fn_sig) = fn_sig()
+                    && fn_sig.skip_binder().abi() != abi::Abi::Rust
+                {
+                    struct_span_err!(tcx.sess, attr.span, E0737, "`#[track_caller]` requires Rust ABI")
+                        .emit();
+                }
+                if tcx.is_closure(did.to_def_id()) && !tcx.features().closure_track_caller {
+                    feature_err(
                         &tcx.sess.parse_sess,
-                        sym::target_feature_11,
+                        sym::closure_track_caller,
                         attr.span,
-                        "`#[target_feature(..)]` can only be applied to `unsafe` functions",
-                    );
-                    err.span_label(tcx.def_span(did), "not an `unsafe` function");
-                    err.emit();
-                } else {
-                    check_target_feature_trait_unsafe(tcx, did, attr.span);
+                        "`#[track_caller]` on closures is currently unstable",
+                    )
+                    .emit();
                 }
+                codegen_fn_attrs.flags |= CodegenFnAttrFlags::TRACK_CALLER
             }
-            from_target_feature(
-                tcx,
-                attr,
-                supported_target_features,
-                &mut codegen_fn_attrs.target_features,
-            );
-        } else if attr.has_name(sym::linkage) {
-            if let Some(val) = attr.value_str() {
-                let linkage = Some(linkage_by_name(tcx, did, val.as_str()));
-                if tcx.is_foreign_item(did) {
-                    codegen_fn_attrs.import_linkage = linkage;
-                } else {
-                    codegen_fn_attrs.linkage = linkage;
+            sym::export_name => {
+                if let Some(s) = attr.value_str() {
+                    if s.as_str().contains('\0') {
+                        // `#[export_name = ...]` will be converted to a null-terminated string,
+                        // so it may not contain any null characters.
+                        struct_span_err!(
+                            tcx.sess,
+                            attr.span,
+                            E0648,
+                            "`export_name` may not contain null characters"
+                        )
+                        .emit();
+                    }
+                    codegen_fn_attrs.export_name = Some(s);
                 }
             }
-        } else if attr.has_name(sym::link_section) {
-            if let Some(val) = attr.value_str() {
-                if val.as_str().bytes().any(|b| b == 0) {
-                    let msg = format!(
-                        "illegal null byte in link_section \
-                         value: `{}`",
-                        &val
-                    );
-                    tcx.sess.span_err(attr.span, &msg);
-                } else {
-                    codegen_fn_attrs.link_section = Some(val);
+            sym::target_feature => {
+                if !tcx.is_closure(did.to_def_id())
+                    && let Some(fn_sig) = fn_sig()
+                    && fn_sig.skip_binder().unsafety() == hir::Unsafety::Normal
+                {
+                    if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc {
+                        // The `#[target_feature]` attribute is allowed on
+                        // WebAssembly targets on all functions, including safe
+                        // ones. Other targets require that `#[target_feature]` is
+                        // only applied to unsafe functions (pending the
+                        // `target_feature_11` feature) because on most targets
+                        // execution of instructions that are not supported is
+                        // considered undefined behavior. For WebAssembly which is a
+                        // 100% safe target at execution time it's not possible to
+                        // execute undefined instructions, and even if a future
+                        // feature was added in some form for this it would be a
+                        // deterministic trap. There is no undefined behavior when
+                        // executing WebAssembly so `#[target_feature]` is allowed
+                        // on safe functions (but again, only for WebAssembly)
+                        //
+                        // Note that this is also allowed if `actually_rustdoc` so
+                        // if a target is documenting some wasm-specific code then
+                        // it's not spuriously denied.
+                        //
+                        // This exception needs to be kept in sync with allowing
+                        // `#[target_feature]` on `main` and `start`.
+                    } else if !tcx.features().target_feature_11 {
+                        let mut err = feature_err(
+                            &tcx.sess.parse_sess,
+                            sym::target_feature_11,
+                            attr.span,
+                            "`#[target_feature(..)]` can only be applied to `unsafe` functions",
+                        );
+                        err.span_label(tcx.def_span(did), "not an `unsafe` function");
+                        err.emit();
+                    } else {
+                        check_target_feature_trait_unsafe(tcx, did, attr.span);
+                    }
                 }
+                from_target_feature(
+                    tcx,
+                    attr,
+                    supported_target_features,
+                    &mut codegen_fn_attrs.target_features,
+                );
             }
-        } else if attr.has_name(sym::link_name) {
-            codegen_fn_attrs.link_name = attr.value_str();
-        } else if attr.has_name(sym::link_ordinal) {
-            link_ordinal_span = Some(attr.span);
-            if let ordinal @ Some(_) = check_link_ordinal(tcx, attr) {
-                codegen_fn_attrs.link_ordinal = ordinal;
+            sym::linkage => {
+                if let Some(val) = attr.value_str() {
+                    let linkage = Some(linkage_by_name(tcx, did, val.as_str()));
+                    if tcx.is_foreign_item(did) {
+                        codegen_fn_attrs.import_linkage = linkage;
+                    } else {
+                        codegen_fn_attrs.linkage = linkage;
+                    }
+                }
             }
-        } else if attr.has_name(sym::no_sanitize) {
-            no_sanitize_span = Some(attr.span);
-            if let Some(list) = attr.meta_item_list() {
-                for item in list.iter() {
-                    if item.has_name(sym::address) {
-                        codegen_fn_attrs.no_sanitize |=
-                            SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS;
-                    } else if item.has_name(sym::cfi) {
-                        codegen_fn_attrs.no_sanitize |= SanitizerSet::CFI;
-                    } else if item.has_name(sym::kcfi) {
-                        codegen_fn_attrs.no_sanitize |= SanitizerSet::KCFI;
-                    } else if item.has_name(sym::memory) {
-                        codegen_fn_attrs.no_sanitize |= SanitizerSet::MEMORY;
-                    } else if item.has_name(sym::memtag) {
-                        codegen_fn_attrs.no_sanitize |= SanitizerSet::MEMTAG;
-                    } else if item.has_name(sym::shadow_call_stack) {
-                        codegen_fn_attrs.no_sanitize |= SanitizerSet::SHADOWCALLSTACK;
-                    } else if item.has_name(sym::thread) {
-                        codegen_fn_attrs.no_sanitize |= SanitizerSet::THREAD;
-                    } else if item.has_name(sym::hwaddress) {
-                        codegen_fn_attrs.no_sanitize |= SanitizerSet::HWADDRESS;
+            sym::link_section => {
+                if let Some(val) = attr.value_str() {
+                    if val.as_str().bytes().any(|b| b == 0) {
+                        let msg = format!("illegal null byte in link_section value: `{}`", &val);
+                        tcx.sess.span_err(attr.span, &msg);
                     } else {
-                        tcx.sess
-                            .struct_span_err(item.span(), "invalid argument for `no_sanitize`")
-                            .note("expected one of: `address`, `cfi`, `hwaddress`, `kcfi`, `memory`, `memtag`, `shadow-call-stack`, or `thread`")
-                            .emit();
+                        codegen_fn_attrs.link_section = Some(val);
                     }
                 }
             }
-        } else if attr.has_name(sym::instruction_set) {
-            codegen_fn_attrs.instruction_set = attr.meta_item_list().and_then(|l| match &l[..] {
-                [NestedMetaItem::MetaItem(set)] => {
-                    let segments =
-                        set.path.segments.iter().map(|x| x.ident.name).collect::<Vec<_>>();
-                    match segments.as_slice() {
-                        [sym::arm, sym::a32] | [sym::arm, sym::t32] => {
-                            if !tcx.sess.target.has_thumb_interworking {
-                                struct_span_err!(
-                                    tcx.sess.diagnostic(),
-                                    attr.span,
-                                    E0779,
-                                    "target does not support `#[instruction_set]`"
-                                )
-                                .emit();
-                                None
-                            } else if segments[1] == sym::a32 {
-                                Some(InstructionSetAttr::ArmA32)
-                            } else if segments[1] == sym::t32 {
-                                Some(InstructionSetAttr::ArmT32)
-                            } else {
-                                unreachable!()
+            sym::link_name => codegen_fn_attrs.link_name = attr.value_str(),
+            sym::link_ordinal => {
+                link_ordinal_span = Some(attr.span);
+                if let ordinal @ Some(_) = check_link_ordinal(tcx, attr) {
+                    codegen_fn_attrs.link_ordinal = ordinal;
+                }
+            }
+            sym::no_sanitize => {
+                no_sanitize_span = Some(attr.span);
+                if let Some(list) = attr.meta_item_list() {
+                    for item in list.iter() {
+                        match item.name_or_empty() {
+                            sym::address => {
+                                codegen_fn_attrs.no_sanitize |=
+                                    SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS
+                            }
+                            sym::cfi => codegen_fn_attrs.no_sanitize |= SanitizerSet::CFI,
+                            sym::kcfi => codegen_fn_attrs.no_sanitize |= SanitizerSet::KCFI,
+                            sym::memory => codegen_fn_attrs.no_sanitize |= SanitizerSet::MEMORY,
+                            sym::memtag => codegen_fn_attrs.no_sanitize |= SanitizerSet::MEMTAG,
+                            sym::shadow_call_stack => {
+                                codegen_fn_attrs.no_sanitize |= SanitizerSet::SHADOWCALLSTACK
+                            }
+                            sym::thread => codegen_fn_attrs.no_sanitize |= SanitizerSet::THREAD,
+                            sym::hwaddress => {
+                                codegen_fn_attrs.no_sanitize |= SanitizerSet::HWADDRESS
+                            }
+                            _ => {
+                                tcx.sess
+                                    .struct_span_err(item.span(), "invalid argument for `no_sanitize`")
+                                    .note("expected one of: `address`, `cfi`, `hwaddress`, `kcfi`, `memory`, `memtag`, `shadow-call-stack`, or `thread`")
+                                    .emit();
                             }
-                        }
-                        _ => {
-                            struct_span_err!(
-                                tcx.sess.diagnostic(),
-                                attr.span,
-                                E0779,
-                                "invalid instruction set specified",
-                            )
-                            .emit();
-                            None
                         }
                     }
                 }
-                [] => {
-                    struct_span_err!(
-                        tcx.sess.diagnostic(),
-                        attr.span,
-                        E0778,
-                        "`#[instruction_set]` requires an argument"
-                    )
-                    .emit();
-                    None
-                }
-                _ => {
-                    struct_span_err!(
-                        tcx.sess.diagnostic(),
-                        attr.span,
-                        E0779,
-                        "cannot specify more than one instruction set"
-                    )
-                    .emit();
-                    None
-                }
-            })
-        } else if attr.has_name(sym::repr) {
-            codegen_fn_attrs.alignment = match attr.meta_item_list() {
-                Some(items) => match items.as_slice() {
-                    [item] => match item.name_value_literal() {
-                        Some((sym::align, literal)) => {
-                            let alignment = rustc_attr::parse_alignment(&literal.kind);
-
-                            match alignment {
-                                Ok(align) => Some(align),
-                                Err(msg) => {
+            }
+            sym::instruction_set => {
+                codegen_fn_attrs.instruction_set =
+                    attr.meta_item_list().and_then(|l| match &l[..] {
+                        [NestedMetaItem::MetaItem(set)] => {
+                            let segments =
+                                set.path.segments.iter().map(|x| x.ident.name).collect::<Vec<_>>();
+                            match segments.as_slice() {
+                                [sym::arm, sym::a32] | [sym::arm, sym::t32] => {
+                                    if !tcx.sess.target.has_thumb_interworking {
+                                        struct_span_err!(
+                                            tcx.sess.diagnostic(),
+                                            attr.span,
+                                            E0779,
+                                            "target does not support `#[instruction_set]`"
+                                        )
+                                        .emit();
+                                        None
+                                    } else if segments[1] == sym::a32 {
+                                        Some(InstructionSetAttr::ArmA32)
+                                    } else if segments[1] == sym::t32 {
+                                        Some(InstructionSetAttr::ArmT32)
+                                    } else {
+                                        unreachable!()
+                                    }
+                                }
+                                _ => {
                                     struct_span_err!(
                                         tcx.sess.diagnostic(),
                                         attr.span,
-                                        E0589,
-                                        "invalid `repr(align)` attribute: {}",
-                                        msg
+                                        E0779,
+                                        "invalid instruction set specified",
                                     )
                                     .emit();
-
                                     None
                                 }
                             }
                         }
-                        _ => None,
-                    },
-                    [] => None,
-                    _ => None,
-                },
-                None => None,
-            };
+                        [] => {
+                            struct_span_err!(
+                                tcx.sess.diagnostic(),
+                                attr.span,
+                                E0778,
+                                "`#[instruction_set]` requires an argument"
+                            )
+                            .emit();
+                            None
+                        }
+                        _ => {
+                            struct_span_err!(
+                                tcx.sess.diagnostic(),
+                                attr.span,
+                                E0779,
+                                "cannot specify more than one instruction set"
+                            )
+                            .emit();
+                            None
+                        }
+                    })
+            }
+            sym::repr => {
+                codegen_fn_attrs.alignment = if let Some(items) = attr.meta_item_list()
+                    && let [item] = items.as_slice()
+                    && let Some((sym::align, literal)) = item.name_value_literal()
+                {
+                    rustc_attr::parse_alignment(&literal.kind).map_err(|msg| {
+                        struct_span_err!(
+                            tcx.sess.diagnostic(),
+                            attr.span,
+                            E0589,
+                            "invalid `repr(align)` attribute: {}",
+                            msg
+                        )
+                        .emit();
+                    })
+                    .ok()
+                } else {
+                    None
+                };
+            }
+            _ => {}
         }
     }
 
diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs
index 1a6495cb1..1ea130400 100644
--- a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs
+++ b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs
@@ -1,6 +1,6 @@
 pub use super::ffi::*;
 
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::mir::coverage::{
     CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId,
     InjectedExpressionIndex, MappedExpressionIndex, Op,
@@ -205,7 +205,7 @@ impl<'tcx> FunctionCoverage<'tcx> {
         // `expression_index`s lower than the referencing `Expression`. Therefore, it is
         // reasonable to look up the new index of an expression operand while the `new_indexes`
         // vector is only complete up to the current `ExpressionIndex`.
-        let id_to_counter = |new_indexes: &IndexVec<
+        let id_to_counter = |new_indexes: &IndexSlice<
             InjectedExpressionIndex,
             Option<MappedExpressionIndex>,
         >,
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index 6dea7496f..66e7e314f 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -405,8 +405,8 @@ pub struct MsvcMissingLinker;
 pub struct CheckInstalledVisualStudio;
 
 #[derive(Diagnostic)]
-#[diag(codegen_ssa_unsufficient_vs_code_product)]
-pub struct UnsufficientVSCodeProduct;
+#[diag(codegen_ssa_insufficient_vs_code_product)]
+pub struct InsufficientVSCodeProduct;
 
 #[derive(Diagnostic)]
 #[diag(codegen_ssa_processing_dymutil_failed)]
diff --git a/compiler/rustc_codegen_ssa/src/glue.rs b/compiler/rustc_codegen_ssa/src/glue.rs
index 0f6e6032f..c34f1dbf8 100644
--- a/compiler/rustc_codegen_ssa/src/glue.rs
+++ b/compiler/rustc_codegen_ssa/src/glue.rs
@@ -46,7 +46,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 // NOTE: ideally, we want the effects of both `unchecked_smul` and `unchecked_umul`
                 // (resulting in `mul nsw nuw` in LLVM IR), since we know that the multiplication
                 // cannot signed wrap, and that both operands are non-negative. But at the time of writing,
-                // `BuilderMethods` can't do this, and it doesn't seem to enable any further optimizations.
+                // the `LLVM-C` binding can't do this, and it doesn't seem to enable any further optimizations.
                 bx.unchecked_smul(info.unwrap(), bx.const_usize(unit.size.bytes())),
                 bx.const_usize(unit.align.abi.bytes()),
             )
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index ebe9e50ff..0ab12314b 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -5,7 +5,6 @@
 #![feature(int_roundings)]
 #![feature(let_chains)]
 #![feature(never_type)]
-#![feature(once_cell)]
 #![feature(strict_provenance)]
 #![feature(try_blocks)]
 #![recursion_limit = "256"]
@@ -56,7 +55,7 @@ pub mod mono_item;
 pub mod target_features;
 pub mod traits;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub struct ModuleCodegen<M> {
     /// The name of the module. When the crate may be saved between
@@ -118,7 +117,7 @@ bitflags::bitflags! {
 #[derive(Clone, Debug, Encodable, Decodable, HashStable)]
 pub struct NativeLib {
     pub kind: NativeLibKind,
-    pub name: Option<Symbol>,
+    pub name: Symbol,
     pub filename: Option<Symbol>,
     pub cfg: Option<ast::MetaItem>,
     pub verbatim: bool,
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index 95aad10fd..f43f1d64a 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -5,7 +5,7 @@ use super::FunctionCx;
 use crate::traits::*;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_index::bit_set::BitSet;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::mir::traversal;
 use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::{self, Location, TerminatorKind};
@@ -232,7 +232,6 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
             | PlaceContext::NonMutatingUse(
                 NonMutatingUseContext::Inspect
                 | NonMutatingUseContext::SharedBorrow
-                | NonMutatingUseContext::UniqueBorrow
                 | NonMutatingUseContext::ShallowBorrow
                 | NonMutatingUseContext::AddressOf
                 | NonMutatingUseContext::Projection,
@@ -277,14 +276,14 @@ impl CleanupKind {
 /// Recover that structure in an analyze pass.
 pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKind> {
     fn discover_masters<'tcx>(
-        result: &mut IndexVec<mir::BasicBlock, CleanupKind>,
+        result: &mut IndexSlice<mir::BasicBlock, CleanupKind>,
         mir: &mir::Body<'tcx>,
     ) {
         for (bb, data) in mir.basic_blocks.iter_enumerated() {
             match data.terminator().kind {
                 TerminatorKind::Goto { .. }
                 | TerminatorKind::Resume
-                | TerminatorKind::Abort
+                | TerminatorKind::Terminate
                 | TerminatorKind::Return
                 | TerminatorKind::GeneratorDrop
                 | TerminatorKind::Unreachable
@@ -292,12 +291,11 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
                 | TerminatorKind::Yield { .. }
                 | TerminatorKind::FalseEdge { .. }
                 | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ }
-                TerminatorKind::Call { cleanup: unwind, .. }
-                | TerminatorKind::InlineAsm { cleanup: unwind, .. }
-                | TerminatorKind::Assert { cleanup: unwind, .. }
-                | TerminatorKind::DropAndReplace { unwind, .. }
+                TerminatorKind::Call { unwind, .. }
+                | TerminatorKind::InlineAsm { unwind, .. }
+                | TerminatorKind::Assert { unwind, .. }
                 | TerminatorKind::Drop { unwind, .. } => {
-                    if let Some(unwind) = unwind {
+                    if let mir::UnwindAction::Cleanup(unwind) = unwind {
                         debug!(
                             "cleanup_kinds: {:?}/{:?} registering {:?} as funclet",
                             bb, data, unwind
@@ -309,7 +307,10 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
         }
     }
 
-    fn propagate<'tcx>(result: &mut IndexVec<mir::BasicBlock, CleanupKind>, mir: &mir::Body<'tcx>) {
+    fn propagate<'tcx>(
+        result: &mut IndexSlice<mir::BasicBlock, CleanupKind>,
+        mir: &mir::Body<'tcx>,
+    ) {
         let mut funclet_succs = IndexVec::from_elem(None, &mir.basic_blocks);
 
         let mut set_successor = |funclet: mir::BasicBlock, succ| match funclet_succs[funclet] {
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 57a19a4ab..dd8697781 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -16,7 +16,7 @@ use rustc_index::vec::Idx;
 use rustc_middle::mir::{self, AssertKind, SwitchTargets};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
-use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt};
+use rustc_middle::ty::{self, Instance, Ty};
 use rustc_session::config::OptLevel;
 use rustc_span::source_map::Span;
 use rustc_span::{sym, Symbol};
@@ -147,7 +147,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     }
 
     /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional
-    /// return destination `destination` and the cleanup function `cleanup`.
+    /// return destination `destination` and the unwind action `unwind`.
     fn do_call<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
         fx: &mut FunctionCx<'a, 'tcx, Bx>,
@@ -156,7 +156,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         fn_ptr: Bx::Value,
         llargs: &[Bx::Value],
         destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>,
-        cleanup: Option<mir::BasicBlock>,
+        mut unwind: mir::UnwindAction,
         copied_constant_arguments: &[PlaceRef<'tcx, <Bx as BackendTypes>::Value>],
         mergeable_succ: bool,
     ) -> MergingSucc {
@@ -164,23 +164,23 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         // do an invoke, otherwise do a call.
         let fn_ty = bx.fn_decl_backend_type(&fn_abi);
 
-        let unwind_block = if let Some(cleanup) = cleanup.filter(|_| fn_abi.can_unwind) {
-            Some(self.llbb_with_cleanup(fx, cleanup))
-        } else if fx.mir[self.bb].is_cleanup
-            && fn_abi.can_unwind
-            && !base::wants_msvc_seh(fx.cx.tcx().sess)
-        {
-            // Exception must not propagate out of the execution of a cleanup (doing so
-            // can cause undefined behaviour). We insert a double unwind guard for
-            // functions that can potentially unwind to protect against this.
-            //
-            // This is not necessary for SEH which does not use successive unwinding
-            // like Itanium EH. EH frames in SEH are different from normal function
-            // frames and SEH will abort automatically if an exception tries to
-            // propagate out from cleanup.
-            Some(fx.double_unwind_guard())
-        } else {
-            None
+        if !fn_abi.can_unwind {
+            unwind = mir::UnwindAction::Unreachable;
+        }
+
+        let unwind_block = match unwind {
+            mir::UnwindAction::Cleanup(cleanup) => Some(self.llbb_with_cleanup(fx, cleanup)),
+            mir::UnwindAction::Continue => None,
+            mir::UnwindAction::Unreachable => None,
+            mir::UnwindAction::Terminate => {
+                if fx.mir[self.bb].is_cleanup && base::wants_msvc_seh(fx.cx.tcx().sess) {
+                    // SEH will abort automatically if an exception tries to
+                    // propagate out from cleanup.
+                    None
+                } else {
+                    Some(fx.terminate_block())
+                }
+            }
         };
 
         if let Some(unwind_block) = unwind_block {
@@ -234,7 +234,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         }
     }
 
-    /// Generates inline assembly with optional `destination` and `cleanup`.
+    /// Generates inline assembly with optional `destination` and `unwind`.
     fn do_inlineasm<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
         fx: &mut FunctionCx<'a, 'tcx, Bx>,
@@ -244,11 +244,18 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         options: InlineAsmOptions,
         line_spans: &[Span],
         destination: Option<mir::BasicBlock>,
-        cleanup: Option<mir::BasicBlock>,
+        unwind: mir::UnwindAction,
         instance: Instance<'_>,
         mergeable_succ: bool,
     ) -> MergingSucc {
-        if let Some(cleanup) = cleanup {
+        let unwind_target = match unwind {
+            mir::UnwindAction::Cleanup(cleanup) => Some(self.llbb_with_cleanup(fx, cleanup)),
+            mir::UnwindAction::Terminate => Some(fx.terminate_block()),
+            mir::UnwindAction::Continue => None,
+            mir::UnwindAction::Unreachable => None,
+        };
+
+        if let Some(cleanup) = unwind_target {
             let ret_llbb = if let Some(target) = destination {
                 fx.llbb(target)
             } else {
@@ -261,7 +268,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
                 options,
                 line_spans,
                 instance,
-                Some((ret_llbb, self.llbb_with_cleanup(fx, cleanup), self.funclet(fx))),
+                Some((ret_llbb, cleanup, self.funclet(fx))),
             );
             MergingSucc::False
         } else {
@@ -397,8 +404,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             PassMode::Cast(cast_ty, _) => {
                 let op = match self.locals[mir::RETURN_PLACE] {
-                    LocalRef::Operand(Some(op)) => op,
-                    LocalRef::Operand(None) => bug!("use of return before def"),
+                    LocalRef::Operand(op) => op,
+                    LocalRef::PendingOperand => bug!("use of return before def"),
                     LocalRef::Place(cg_place) => OperandRef {
                         val: Ref(cg_place.llval, None, cg_place.align),
                         layout: cg_place.layout,
@@ -431,7 +438,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         bx: &mut Bx,
         location: mir::Place<'tcx>,
         target: mir::BasicBlock,
-        unwind: Option<mir::BasicBlock>,
+        unwind: mir::UnwindAction,
         mergeable_succ: bool,
     ) -> MergingSucc {
         let ty = location.ty(self.mir, bx.tcx()).ty;
@@ -552,7 +559,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         expected: bool,
         msg: &mir::AssertMessage<'tcx>,
         target: mir::BasicBlock,
-        cleanup: Option<mir::BasicBlock>,
+        unwind: mir::UnwindAction,
         mergeable_succ: bool,
     ) -> MergingSucc {
         let span = terminator.source_info.span;
@@ -563,15 +570,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // with #[rustc_inherit_overflow_checks] and inlined from
         // another crate (mostly core::num generic/#[inline] fns),
         // while the current crate doesn't use overflow checks.
-        if !bx.cx().check_overflow() {
-            let overflow_not_to_check = match msg {
-                AssertKind::OverflowNeg(..) => true,
-                AssertKind::Overflow(op, ..) => op.is_checkable(),
-                _ => false,
-            };
-            if overflow_not_to_check {
-                const_cond = Some(expected);
-            }
+        if !bx.cx().check_overflow() && msg.is_optional_overflow_check() {
+            const_cond = Some(expected);
         }
 
         // Don't codegen the panic block if success if known.
@@ -607,6 +607,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 // and `#[track_caller]` adds an implicit third argument.
                 (LangItem::PanicBoundsCheck, vec![index, len, location])
             }
+            AssertKind::MisalignedPointerDereference { ref required, ref found } => {
+                let required = self.codegen_operand(bx, required).immediate();
+                let found = self.codegen_operand(bx, found).immediate();
+                // It's `fn panic_bounds_check(index: usize, len: usize)`,
+                // and `#[track_caller]` adds an implicit third argument.
+                (LangItem::PanicMisalignedPointerDereference, vec![required, found, location])
+            }
             _ => {
                 let msg = bx.const_str(msg.description());
                 // It's `pub fn panic(expr: &str)`, with the wide reference being passed
@@ -618,12 +625,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let (fn_abi, llfn) = common::build_langcall(bx, Some(span), lang_item);
 
         // Codegen the actual panic invoke/call.
-        let merging_succ = helper.do_call(self, bx, fn_abi, llfn, &args, None, cleanup, &[], false);
+        let merging_succ = helper.do_call(self, bx, fn_abi, llfn, &args, None, unwind, &[], false);
         assert_eq!(merging_succ, MergingSucc::False);
         MergingSucc::False
     }
 
-    fn codegen_abort_terminator(
+    fn codegen_terminate_terminator(
         &mut self,
         helper: TerminatorCodegenHelper<'tcx>,
         bx: &mut Bx,
@@ -636,7 +643,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let (fn_abi, llfn) = common::build_langcall(bx, Some(span), LangItem::PanicCannotUnwind);
 
         // Codegen the actual panic invoke/call.
-        let merging_succ = helper.do_call(self, bx, fn_abi, llfn, &[], None, None, &[], false);
+        let merging_succ = helper.do_call(
+            self,
+            bx,
+            fn_abi,
+            llfn,
+            &[],
+            None,
+            mir::UnwindAction::Unreachable,
+            &[],
+            false,
+        );
         assert_eq!(merging_succ, MergingSucc::False);
     }
 
@@ -649,7 +666,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         instance: Option<Instance<'tcx>>,
         source_info: mir::SourceInfo,
         target: Option<mir::BasicBlock>,
-        cleanup: Option<mir::BasicBlock>,
+        unwind: mir::UnwindAction,
         mergeable_succ: bool,
     ) -> Option<MergingSucc> {
         // Emit a panic or a no-op for `assert_*` intrinsics.
@@ -696,7 +713,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     llfn,
                     &[msg.0, msg.1],
                     target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)),
-                    cleanup,
+                    unwind,
                     &[],
                     mergeable_succ,
                 )
@@ -719,7 +736,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         args: &[mir::Operand<'tcx>],
         destination: mir::Place<'tcx>,
         target: Option<mir::BasicBlock>,
-        cleanup: Option<mir::BasicBlock>,
+        unwind: mir::UnwindAction,
         fn_span: Span,
         mergeable_succ: bool,
     ) -> MergingSucc {
@@ -776,23 +793,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             None => bx.fn_abi_of_fn_ptr(sig, extra_args),
         };
 
-        if intrinsic == Some(sym::transmute) {
-            return if let Some(target) = target {
-                self.codegen_transmute(bx, &args[0], destination);
-                helper.funclet_br(self, bx, target, mergeable_succ)
-            } else {
-                // If we are trying to transmute to an uninhabited type,
-                // it is likely there is no allotted destination. In fact,
-                // transmuting to an uninhabited type is UB, which means
-                // we can do what we like. Here, we declare that transmuting
-                // into an uninhabited type is impossible, so anything following
-                // it must be unreachable.
-                assert_eq!(fn_abi.ret.layout.abi, abi::Abi::Uninhabited);
-                bx.unreachable();
-                MergingSucc::False
-            };
-        }
-
         if let Some(merging_succ) = self.codegen_panic_intrinsic(
             &helper,
             bx,
@@ -800,7 +800,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             instance,
             source_info,
             target,
-            cleanup,
+            unwind,
             mergeable_succ,
         ) {
             return merging_succ;
@@ -835,7 +835,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         match intrinsic {
             None | Some(sym::drop_in_place) => {}
-            Some(sym::copy_nonoverlapping) => unreachable!(),
             Some(intrinsic) => {
                 let dest = match ret_dest {
                     _ if fn_abi.ret.is_indirect() => llargs[0],
@@ -924,7 +923,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         //
                         // This is also relevant for `Pin<&mut Self>`, where we need to peel the `Pin`.
                         'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
-                            && !op.layout.ty.is_region_ptr()
+                            && !op.layout.ty.is_ref()
                         {
                             for i in 0..op.layout.fields.count() {
                                 let field = op.extract_field(bx, i);
@@ -966,7 +965,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     Immediate(_) => {
                         // See comment above explaining why we peel these newtypes
                         'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
-                            && !op.layout.ty.is_region_ptr()
+                            && !op.layout.ty.is_ref()
                         {
                             for i in 0..op.layout.fields.count() {
                                 let field = op.extract_field(bx, i);
@@ -1082,7 +1081,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 fn_ptr,
                 &llargs,
                 target.as_ref().map(|&target| (ret_dest, target)),
-                cleanup,
+                unwind,
                 &copied_constant_arguments,
                 false,
             );
@@ -1102,7 +1101,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             fn_ptr,
             &llargs,
             target.as_ref().map(|&target| (ret_dest, target)),
-            cleanup,
+            unwind,
             &copied_constant_arguments,
             mergeable_succ,
         )
@@ -1118,7 +1117,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         options: ast::InlineAsmOptions,
         line_spans: &[Span],
         destination: Option<mir::BasicBlock>,
-        cleanup: Option<mir::BasicBlock>,
+        unwind: mir::UnwindAction,
         instance: Instance<'_>,
         mergeable_succ: bool,
     ) -> MergingSucc {
@@ -1182,7 +1181,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             options,
             line_spans,
             destination,
-            cleanup,
+            unwind,
             instance,
             mergeable_succ,
         )
@@ -1264,8 +1263,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 MergingSucc::False
             }
 
-            mir::TerminatorKind::Abort => {
-                self.codegen_abort_terminator(helper, bx, terminator);
+            mir::TerminatorKind::Terminate => {
+                self.codegen_terminate_terminator(helper, bx, terminator);
                 MergingSucc::False
             }
 
@@ -1292,7 +1291,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_drop_terminator(helper, bx, place, target, unwind, mergeable_succ())
             }
 
-            mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, cleanup } => self
+            mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, unwind } => self
                 .codegen_assert_terminator(
                     helper,
                     bx,
@@ -1301,20 +1300,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     expected,
                     msg,
                     target,
-                    cleanup,
+                    unwind,
                     mergeable_succ(),
                 ),
 
-            mir::TerminatorKind::DropAndReplace { .. } => {
-                bug!("undesugared DropAndReplace in codegen: {:?}", terminator);
-            }
-
             mir::TerminatorKind::Call {
                 ref func,
                 ref args,
                 destination,
                 target,
-                cleanup,
+                unwind,
                 from_hir_call: _,
                 fn_span,
             } => self.codegen_call_terminator(
@@ -1325,7 +1320,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 args,
                 destination,
                 target,
-                cleanup,
+                unwind,
                 fn_span,
                 mergeable_succ(),
             ),
@@ -1342,7 +1337,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 options,
                 line_spans,
                 destination,
-                cleanup,
+                unwind,
             } => self.codegen_asm_terminator(
                 helper,
                 bx,
@@ -1352,7 +1347,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 options,
                 line_spans,
                 destination,
-                cleanup,
+                unwind,
                 self.instance,
                 mergeable_succ(),
             ),
@@ -1486,7 +1481,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) -> OperandRef<'tcx, Bx::Value> {
         let tcx = bx.tcx();
 
-        let mut span_to_caller_location = |span: Span| {
+        let mut span_to_caller_location = |mut span: Span| {
+            // Remove `Inlined` marks as they pollute `expansion_cause`.
+            while span.is_inlined() {
+                span.remove_mark();
+            }
             let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
             let caller = tcx.sess.source_map().lookup_char_pos(topmost.lo());
             let const_loc = tcx.const_caller_location((
@@ -1554,62 +1553,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     fn landing_pad_for_uncached(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock {
         let llbb = self.llbb(bb);
         if base::wants_msvc_seh(self.cx.sess()) {
-            let funclet;
-            let ret_llbb;
-            match self.mir[bb].terminator.as_ref().map(|t| &t.kind) {
-                // This is a basic block that we're aborting the program for,
-                // notably in an `extern` function. These basic blocks are inserted
-                // so that we assert that `extern` functions do indeed not panic,
-                // and if they do we abort the process.
-                //
-                // On MSVC these are tricky though (where we're doing funclets). If
-                // we were to do a cleanuppad (like below) the normal functions like
-                // `longjmp` would trigger the abort logic, terminating the
-                // program. Instead we insert the equivalent of `catch(...)` for C++
-                // which magically doesn't trigger when `longjmp` files over this
-                // frame.
-                //
-                // Lots more discussion can be found on #48251 but this codegen is
-                // modeled after clang's for:
-                //
-                //      try {
-                //          foo();
-                //      } catch (...) {
-                //          bar();
-                //      }
-                Some(&mir::TerminatorKind::Abort) => {
-                    let cs_llbb =
-                        Bx::append_block(self.cx, self.llfn, &format!("cs_funclet{:?}", bb));
-                    let cp_llbb =
-                        Bx::append_block(self.cx, self.llfn, &format!("cp_funclet{:?}", bb));
-                    ret_llbb = cs_llbb;
-
-                    let mut cs_bx = Bx::build(self.cx, cs_llbb);
-                    let cs = cs_bx.catch_switch(None, None, &[cp_llbb]);
-
-                    // The "null" here is actually a RTTI type descriptor for the
-                    // C++ personality function, but `catch (...)` has no type so
-                    // it's null. The 64 here is actually a bitfield which
-                    // represents that this is a catch-all block.
-                    let mut cp_bx = Bx::build(self.cx, cp_llbb);
-                    let null = cp_bx.const_null(
-                        cp_bx.type_i8p_ext(cp_bx.cx().data_layout().instruction_address_space),
-                    );
-                    let sixty_four = cp_bx.const_i32(64);
-                    funclet = cp_bx.catch_pad(cs, &[null, sixty_four, null]);
-                    cp_bx.br(llbb);
-                }
-                _ => {
-                    let cleanup_llbb =
-                        Bx::append_block(self.cx, self.llfn, &format!("funclet_{:?}", bb));
-                    ret_llbb = cleanup_llbb;
-                    let mut cleanup_bx = Bx::build(self.cx, cleanup_llbb);
-                    funclet = cleanup_bx.cleanup_pad(None, &[]);
-                    cleanup_bx.br(llbb);
-                }
-            }
+            let cleanup_bb = Bx::append_block(self.cx, self.llfn, &format!("funclet_{:?}", bb));
+            let mut cleanup_bx = Bx::build(self.cx, cleanup_bb);
+            let funclet = cleanup_bx.cleanup_pad(None, &[]);
+            cleanup_bx.br(llbb);
             self.funclets[bb] = Some(funclet);
-            ret_llbb
+            cleanup_bb
         } else {
             let cleanup_llbb = Bx::append_block(self.cx, self.llfn, "cleanup");
             let mut cleanup_bx = Bx::build(self.cx, cleanup_llbb);
@@ -1636,26 +1585,68 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         })
     }
 
-    fn double_unwind_guard(&mut self) -> Bx::BasicBlock {
-        self.double_unwind_guard.unwrap_or_else(|| {
-            assert!(!base::wants_msvc_seh(self.cx.sess()));
+    fn terminate_block(&mut self) -> Bx::BasicBlock {
+        self.terminate_block.unwrap_or_else(|| {
+            let funclet;
+            let llbb;
+            let mut bx;
+            if base::wants_msvc_seh(self.cx.sess()) {
+                // This is a basic block that we're aborting the program for,
+                // notably in an `extern` function. These basic blocks are inserted
+                // so that we assert that `extern` functions do indeed not panic,
+                // and if they do we abort the process.
+                //
+                // On MSVC these are tricky though (where we're doing funclets). If
+                // we were to do a cleanuppad (like below) the normal functions like
+                // `longjmp` would trigger the abort logic, terminating the
+                // program. Instead we insert the equivalent of `catch(...)` for C++
+                // which magically doesn't trigger when `longjmp` files over this
+                // frame.
+                //
+                // Lots more discussion can be found on #48251 but this codegen is
+                // modeled after clang's for:
+                //
+                //      try {
+                //          foo();
+                //      } catch (...) {
+                //          bar();
+                //      }
+                llbb = Bx::append_block(self.cx, self.llfn, "cs_terminate");
+                let cp_llbb = Bx::append_block(self.cx, self.llfn, "cp_terminate");
+
+                let mut cs_bx = Bx::build(self.cx, llbb);
+                let cs = cs_bx.catch_switch(None, None, &[cp_llbb]);
+
+                // The "null" here is actually a RTTI type descriptor for the
+                // C++ personality function, but `catch (...)` has no type so
+                // it's null. The 64 here is actually a bitfield which
+                // represents that this is a catch-all block.
+                bx = Bx::build(self.cx, cp_llbb);
+                let null =
+                    bx.const_null(bx.type_i8p_ext(bx.cx().data_layout().instruction_address_space));
+                let sixty_four = bx.const_i32(64);
+                funclet = Some(bx.catch_pad(cs, &[null, sixty_four, null]));
+            } else {
+                llbb = Bx::append_block(self.cx, self.llfn, "terminate");
+                bx = Bx::build(self.cx, llbb);
 
-            let llbb = Bx::append_block(self.cx, self.llfn, "abort");
-            let mut bx = Bx::build(self.cx, llbb);
-            self.set_debug_loc(&mut bx, mir::SourceInfo::outermost(self.mir.span));
+                let llpersonality = self.cx.eh_personality();
+                bx.cleanup_landing_pad(llpersonality);
 
-            let llpersonality = self.cx.eh_personality();
-            bx.cleanup_landing_pad(llpersonality);
+                funclet = None;
+            }
+
+            self.set_debug_loc(&mut bx, mir::SourceInfo::outermost(self.mir.span));
 
             let (fn_abi, fn_ptr) = common::build_langcall(&bx, None, LangItem::PanicCannotUnwind);
             let fn_ty = bx.fn_decl_backend_type(&fn_abi);
 
-            let llret = bx.call(fn_ty, Some(&fn_abi), fn_ptr, &[], None);
+            let llret = bx.call(fn_ty, Some(&fn_abi), fn_ptr, &[], funclet.as_ref());
             bx.do_not_inline(llret);
 
             bx.unreachable();
 
-            self.double_unwind_guard = Some(llbb);
+            self.terminate_block = Some(llbb);
             llbb
         })
     }
@@ -1698,7 +1689,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             match self.locals[index] {
                 LocalRef::Place(dest) => dest,
                 LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
-                LocalRef::Operand(None) => {
+                LocalRef::PendingOperand => {
                     // Handle temporary places, specifically `Operand` ones, as
                     // they don't have `alloca`s.
                     return if fn_ret.is_indirect() {
@@ -1719,7 +1710,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         ReturnDest::DirectOperand(index)
                     };
                 }
-                LocalRef::Operand(Some(_)) => {
+                LocalRef::Operand(_) => {
                     bug!("place local already assigned to");
                 }
             }
@@ -1746,71 +1737,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         }
     }
 
-    fn codegen_transmute(&mut self, bx: &mut Bx, src: &mir::Operand<'tcx>, dst: mir::Place<'tcx>) {
-        if let Some(index) = dst.as_local() {
-            match self.locals[index] {
-                LocalRef::Place(place) => self.codegen_transmute_into(bx, src, place),
-                LocalRef::UnsizedPlace(_) => bug!("transmute must not involve unsized locals"),
-                LocalRef::Operand(None) => {
-                    let dst_layout = bx.layout_of(self.monomorphized_place_ty(dst.as_ref()));
-                    assert!(!dst_layout.ty.has_erasable_regions());
-                    let place = PlaceRef::alloca(bx, dst_layout);
-                    place.storage_live(bx);
-                    self.codegen_transmute_into(bx, src, place);
-                    let op = bx.load_operand(place);
-                    place.storage_dead(bx);
-                    self.locals[index] = LocalRef::Operand(Some(op));
-                    self.debug_introduce_local(bx, index);
-                }
-                LocalRef::Operand(Some(op)) => {
-                    assert!(op.layout.is_zst(), "assigning to initialized SSAtemp");
-                }
-            }
-        } else {
-            let dst = self.codegen_place(bx, dst.as_ref());
-            self.codegen_transmute_into(bx, src, dst);
-        }
-    }
-
-    fn codegen_transmute_into(
-        &mut self,
-        bx: &mut Bx,
-        src: &mir::Operand<'tcx>,
-        dst: PlaceRef<'tcx, Bx::Value>,
-    ) {
-        let src = self.codegen_operand(bx, src);
-
-        // Special-case transmutes between scalars as simple bitcasts.
-        match (src.layout.abi, dst.layout.abi) {
-            (abi::Abi::Scalar(src_scalar), abi::Abi::Scalar(dst_scalar)) => {
-                // HACK(eddyb) LLVM doesn't like `bitcast`s between pointers and non-pointers.
-                let src_is_ptr = matches!(src_scalar.primitive(), abi::Pointer(_));
-                let dst_is_ptr = matches!(dst_scalar.primitive(), abi::Pointer(_));
-                if src_is_ptr == dst_is_ptr {
-                    assert_eq!(src.layout.size, dst.layout.size);
-
-                    // NOTE(eddyb) the `from_immediate` and `to_immediate_scalar`
-                    // conversions allow handling `bool`s the same as `u8`s.
-                    let src = bx.from_immediate(src.immediate());
-                    // LLVM also doesn't like `bitcast`s between pointers in different address spaces.
-                    let src_as_dst = if src_is_ptr {
-                        bx.pointercast(src, bx.backend_type(dst.layout))
-                    } else {
-                        bx.bitcast(src, bx.backend_type(dst.layout))
-                    };
-                    Immediate(bx.to_immediate_scalar(src_as_dst, dst_scalar)).store(bx, dst);
-                    return;
-                }
-            }
-            _ => {}
-        }
-
-        let llty = bx.backend_type(src.layout);
-        let cast_ptr = bx.pointercast(dst.llval, bx.type_ptr_to(llty));
-        let align = src.layout.align.abi.min(dst.align);
-        src.val.store(bx, PlaceRef::new_sized_aligned(cast_ptr, src.layout, align));
-    }
-
     // Stores the return value of a function call into it's final location.
     fn store_return(
         &mut self,
@@ -1827,7 +1753,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             IndirectOperand(tmp, index) => {
                 let op = bx.load_operand(tmp);
                 tmp.storage_dead(bx);
-                self.locals[index] = LocalRef::Operand(Some(op));
+                self.locals[index] = LocalRef::Operand(op);
                 self.debug_introduce_local(bx, index);
             }
             DirectOperand(index) => {
@@ -1842,7 +1768,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 } else {
                     OperandRef::from_immediate_or_packed_pair(bx, llval, ret_abi.layout)
                 };
-                self.locals[index] = LocalRef::Operand(Some(op));
+                self.locals[index] = LocalRef::Operand(op);
                 self.debug_introduce_local(bx, index);
             }
         }
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
index 708f3bc0c..d049bafb8 100644
--- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -8,7 +8,7 @@ use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 use rustc_session::config::DebugInfo;
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{BytePos, Span};
-use rustc_target::abi::{Abi, Size, VariantIdx};
+use rustc_target::abi::{Abi, FieldIdx, Size, VariantIdx};
 
 use super::operand::{OperandRef, OperandValue};
 use super::place::PlaceRef;
@@ -79,7 +79,7 @@ impl<'tcx, S: Copy, L: Copy> DebugScope<S, L> {
 trait DebugInfoOffsetLocation<'tcx, Bx> {
     fn deref(&self, bx: &mut Bx) -> Self;
     fn layout(&self) -> TyAndLayout<'tcx>;
-    fn project_field(&self, bx: &mut Bx, field: mir::Field) -> Self;
+    fn project_field(&self, bx: &mut Bx, field: FieldIdx) -> Self;
     fn downcast(&self, bx: &mut Bx, variant: VariantIdx) -> Self;
 }
 
@@ -94,7 +94,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> DebugInfoOffsetLocation<'tcx, Bx>
         self.layout
     }
 
-    fn project_field(&self, bx: &mut Bx, field: mir::Field) -> Self {
+    fn project_field(&self, bx: &mut Bx, field: FieldIdx) -> Self {
         PlaceRef::project_field(*self, bx, field.index())
     }
 
@@ -116,7 +116,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> DebugInfoOffsetLocation<'tcx, Bx>
         *self
     }
 
-    fn project_field(&self, bx: &mut Bx, field: mir::Field) -> Self {
+    fn project_field(&self, bx: &mut Bx, field: FieldIdx) -> Self {
         self.field(bx.cx(), field.index())
     }
 
@@ -165,11 +165,15 @@ fn calculate_debuginfo_offset<
             mir::ProjectionElem::Downcast(_, variant) => {
                 place = place.downcast(bx, variant);
             }
-            _ => span_bug!(
-                var.source_info.span,
-                "unsupported var debuginfo place `{:?}`",
-                mir::Place { local, projection: var.projection },
-            ),
+            _ => {
+                // Sanity check for `can_use_in_debuginfo`.
+                debug_assert!(!elem.can_use_in_debuginfo());
+                span_bug!(
+                    var.source_info.span,
+                    "unsupported var debuginfo place `{:?}`",
+                    mir::Place { local, projection: var.projection },
+                )
+            }
         }
     }
 
@@ -241,12 +245,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     pub fn debug_introduce_local(&self, bx: &mut Bx, local: mir::Local) {
         let full_debug_info = bx.sess().opts.debuginfo == DebugInfo::Full;
 
-        // FIXME(eddyb) maybe name the return place as `_0` or `return`?
-        if local == mir::RETURN_PLACE && !self.mir.local_decls[mir::RETURN_PLACE].is_user_variable()
-        {
-            return;
-        }
-
         let vars = match &self.per_local_var_debug_info {
             Some(per_local) => &per_local[local],
             None => return,
@@ -303,7 +301,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         let local_ref = &self.locals[local];
 
-        let name = if bx.sess().fewer_names() {
+        // FIXME Should the return place be named?
+        let name = if bx.sess().fewer_names() || local == mir::RETURN_PLACE {
             None
         } else {
             Some(match whole_local_var.or(fallback_var.clone()) {
@@ -317,7 +316,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 LocalRef::Place(place) | LocalRef::UnsizedPlace(place) => {
                     bx.set_var_name(place.llval, name);
                 }
-                LocalRef::Operand(Some(operand)) => match operand.val {
+                LocalRef::Operand(operand) => match operand.val {
                     OperandValue::Ref(x, ..) | OperandValue::Immediate(x) => {
                         bx.set_var_name(x, name);
                     }
@@ -328,7 +327,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         bx.set_var_name(b, &(name.clone() + ".1"));
                     }
                 },
-                LocalRef::Operand(None) => {}
+                LocalRef::PendingOperand => {}
             }
         }
 
@@ -337,9 +336,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         }
 
         let base = match local_ref {
-            LocalRef::Operand(None) => return,
+            LocalRef::PendingOperand => return,
 
-            LocalRef::Operand(Some(operand)) => {
+            LocalRef::Operand(operand) => {
                 // Don't spill operands onto the stack in naked functions.
                 // See: https://github.com/rust-lang/rust/issues/42779
                 let attrs = bx.tcx().codegen_fn_attrs(self.instance.def_id());
@@ -443,11 +442,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let (var_ty, var_kind) = match var.value {
                     mir::VarDebugInfoContents::Place(place) => {
                         let var_ty = self.monomorphized_place_ty(place.as_ref());
-                        let var_kind = if self.mir.local_kind(place.local) == mir::LocalKind::Arg
+                        let var_kind = if let Some(arg_index) = var.argument_index
                             && place.projection.is_empty()
-                            && var.source_info.scope == mir::OUTERMOST_SOURCE_SCOPE
                         {
-                            let arg_index = place.local.index() - 1;
+                            let arg_index = arg_index as usize;
                             if target_is_msvc {
                                 // ScalarPair parameters are spilled to the stack so they need to
                                 // be marked as a `LocalVariable` for MSVC debuggers to visualize
@@ -456,13 +454,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 if let Abi::ScalarPair(_, _) = var_ty_layout.abi {
                                     VariableKind::LocalVariable
                                 } else {
-                                    VariableKind::ArgumentVariable(arg_index + 1)
+                                    VariableKind::ArgumentVariable(arg_index)
                                 }
                             } else {
                                 // FIXME(eddyb) shouldn't `ArgumentVariable` indices be
                                 // offset in closures to account for the hidden environment?
-                                // Also, is this `+ 1` needed at all?
-                                VariableKind::ArgumentVariable(arg_index + 1)
+                                VariableKind::ArgumentVariable(arg_index)
                             }
                         } else {
                             VariableKind::LocalVariable
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index 2ec9fdbf4..3dadb33c9 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -73,8 +73,8 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
     /// Cached unreachable block
     unreachable_block: Option<Bx::BasicBlock>,
 
-    /// Cached double unwind guarding block
-    double_unwind_guard: Option<Bx::BasicBlock>,
+    /// Cached terminate upon unwinding block
+    terminate_block: Option<Bx::BasicBlock>,
 
     /// The location where each MIR arg/var/tmp/ret is stored. This is
     /// usually an `PlaceRef` representing an alloca, but not always:
@@ -123,7 +123,10 @@ enum LocalRef<'tcx, V> {
     /// Every time it is initialized, we have to reallocate the place
     /// and update the fat pointer. That's the reason why it is indirect.
     UnsizedPlace(PlaceRef<'tcx, V>),
-    Operand(Option<OperandRef<'tcx, V>>),
+    /// The backend [`OperandValue`] has already been generated.
+    Operand(OperandRef<'tcx, V>),
+    /// Will be a `Self::Operand` once we get to its definition.
+    PendingOperand,
 }
 
 impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> {
@@ -135,9 +138,9 @@ impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> {
             // Zero-size temporaries aren't always initialized, which
             // doesn't matter because they don't contain data, but
             // we need something in the operand.
-            LocalRef::Operand(Some(OperandRef::new_zst(bx, layout)))
+            LocalRef::Operand(OperandRef::new_zst(bx, layout))
         } else {
-            LocalRef::Operand(None)
+            LocalRef::PendingOperand
         }
     }
 }
@@ -163,7 +166,9 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     let start_llbb = Bx::append_block(cx, llfn, "start");
     let mut start_bx = Bx::build(cx, start_llbb);
 
-    if mir.basic_blocks.iter().any(|bb| bb.is_cleanup) {
+    if mir.basic_blocks.iter().any(|bb| {
+        bb.is_cleanup || matches!(bb.terminator().unwind(), Some(mir::UnwindAction::Terminate))
+    }) {
         start_bx.set_personality_fn(cx.eh_personality());
     }
 
@@ -186,7 +191,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         personality_slot: None,
         cached_llbbs,
         unreachable_block: None,
-        double_unwind_guard: None,
+        terminate_block: None,
         cleanup_kinds,
         landing_pads: IndexVec::from_elem(None, &mir.basic_blocks),
         funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()),
@@ -258,6 +263,10 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     // Apply debuginfo to the newly allocated locals.
     fx.debug_introduce_locals(&mut start_bx);
 
+    // The builders will be created separately for each basic block at `codegen_block`.
+    // So drop the builder of `start_llbb` to avoid having two at the same time.
+    drop(start_bx);
+
     // Codegen the body of each block using reverse postorder
     for (bb, _) in traversal::reverse_postorder(&mir) {
         fx.codegen_block(bb);
@@ -333,7 +342,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 // We don't have to cast or keep the argument in the alloca.
                 // FIXME(eddyb): We should figure out how to use llvm.dbg.value instead
                 // of putting everything in allocas just so we can use llvm.dbg.declare.
-                let local = |op| LocalRef::Operand(Some(op));
+                let local = |op| LocalRef::Operand(op);
                 match arg.mode {
                     PassMode::Ignore => {
                         return local(OperandRef::new_zst(bx, arg.layout));
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index 34a5b638d..b37797fef 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -23,10 +23,26 @@ pub enum OperandValue<V> {
     /// to be valid for the operand's lifetime.
     /// The second value, if any, is the extra data (vtable or length)
     /// which indicates that it refers to an unsized rvalue.
+    ///
+    /// An `OperandValue` has this variant for types which are neither
+    /// `Immediate` nor `Pair`s. The backend value in this variant must be a
+    /// pointer to the *non*-immediate backend type. That pointee type is the
+    /// one returned by [`LayoutTypeMethods::backend_type`].
     Ref(V, Option<V>, Align),
-    /// A single LLVM value.
+    /// A single LLVM immediate value.
+    ///
+    /// An `OperandValue` *must* be this variant for any type for which
+    /// [`LayoutTypeMethods::is_backend_immediate`] returns `true`.
+    /// The backend value in this variant must be the *immediate* backend type,
+    /// as returned by [`LayoutTypeMethods::immediate_backend_type`].
     Immediate(V),
     /// A pair of immediate LLVM values. Used by fat pointers too.
+    ///
+    /// An `OperandValue` *must* be this variant for any type for which
+    /// [`LayoutTypeMethods::is_backend_scalar_pair`] returns `true`.
+    /// The backend values in this variant must be the *immediate* backend types,
+    /// as returned by [`LayoutTypeMethods::scalar_pair_element_backend_type`]
+    /// with `immediate: true`.
     Pair(V, V),
 }
 
@@ -60,7 +76,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
     ) -> OperandRef<'tcx, V> {
         assert!(layout.is_zst());
         OperandRef {
-            val: OperandValue::Immediate(bx.const_undef(bx.immediate_backend_type(layout))),
+            val: OperandValue::Immediate(bx.const_poison(bx.immediate_backend_type(layout))),
             layout,
         }
     }
@@ -145,7 +161,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             let llty = bx.cx().backend_type(self.layout);
             debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty);
             // Reconstruct the immediate aggregate.
-            let mut llpair = bx.cx().const_undef(llty);
+            let mut llpair = bx.cx().const_poison(llty);
             let imm_a = bx.from_immediate(a);
             let imm_b = bx.from_immediate(b);
             llpair = bx.insert_value(llpair, imm_a, 0);
@@ -243,6 +259,31 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 }
 
 impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
+    /// Returns an `OperandValue` that's generally UB to use in any way.
+    ///
+    /// Depending on the `layout`, returns an `Immediate` or `Pair` containing
+    /// poison value(s), or a `Ref` containing a poison pointer.
+    ///
+    /// Supports sized types only.
+    pub fn poison<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+        bx: &mut Bx,
+        layout: TyAndLayout<'tcx>,
+    ) -> OperandValue<V> {
+        assert!(layout.is_sized());
+        if bx.cx().is_backend_immediate(layout) {
+            let ibty = bx.cx().immediate_backend_type(layout);
+            OperandValue::Immediate(bx.const_poison(ibty))
+        } else if bx.cx().is_backend_scalar_pair(layout) {
+            let ibty0 = bx.cx().scalar_pair_element_backend_type(layout, 0, true);
+            let ibty1 = bx.cx().scalar_pair_element_backend_type(layout, 1, true);
+            OperandValue::Pair(bx.const_poison(ibty0), bx.const_poison(ibty1))
+        } else {
+            let bty = bx.cx().backend_type(layout);
+            let ptr_bty = bx.cx().type_ptr_to(bty);
+            OperandValue::Ref(bx.const_poison(ptr_bty), None, layout.align.abi)
+        }
+    }
+
     pub fn store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
         self,
         bx: &mut Bx,
@@ -370,7 +411,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref);
 
         match self.locals[place_ref.local] {
-            LocalRef::Operand(Some(mut o)) => {
+            LocalRef::Operand(mut o) => {
                 // Moves out of scalar and scalar pair fields are trivial.
                 for elem in place_ref.projection.iter() {
                     match elem {
@@ -395,7 +436,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
                 Some(o)
             }
-            LocalRef::Operand(None) => {
+            LocalRef::PendingOperand => {
                 bug!("use of {:?} before def", place_ref);
             }
             LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => {
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index cf02f59f6..a58a61cd5 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -211,10 +211,9 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
     ) -> V {
         let dl = &bx.tcx().data_layout;
         let cast_to_layout = bx.cx().layout_of(cast_to);
-        let cast_to_size = cast_to_layout.layout.size();
         let cast_to = bx.cx().immediate_backend_type(cast_to_layout);
         if self.layout.abi.is_uninhabited() {
-            return bx.cx().const_undef(cast_to);
+            return bx.cx().const_poison(cast_to);
         }
         let (tag_scalar, tag_encoding, tag_field) = match self.layout.variants {
             Variants::Single { index } => {
@@ -261,21 +260,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                     _ => (tag_imm, bx.cx().immediate_backend_type(tag_op.layout)),
                 };
 
-                let tag_size = tag_scalar.size(bx.cx());
-                let max_unsigned = tag_size.unsigned_int_max();
-                let max_signed = tag_size.signed_int_max() as u128;
-                let min_signed = max_signed + 1;
                 let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
-                let niche_end = niche_start.wrapping_add(relative_max as u128) & max_unsigned;
-                let range = tag_scalar.valid_range(bx.cx());
-
-                let sle = |lhs: u128, rhs: u128| -> bool {
-                    // Signed and unsigned comparisons give the same results,
-                    // except that in signed comparisons an integer with the
-                    // sign bit set is less than one with the sign bit clear.
-                    // Toggle the sign bit to do a signed comparison.
-                    (lhs ^ min_signed) <= (rhs ^ min_signed)
-                };
 
                 // We have a subrange `niche_start..=niche_end` inside `range`.
                 // If the value of the tag is inside this subrange, it's a
@@ -291,49 +276,6 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                 //     untagged_variant
                 // }
                 // However, we will likely be able to emit simpler code.
-
-                // Find the least and greatest values in `range`, considered
-                // both as signed and unsigned.
-                let (low_unsigned, high_unsigned) = if range.start <= range.end {
-                    (range.start, range.end)
-                } else {
-                    (0, max_unsigned)
-                };
-                let (low_signed, high_signed) = if sle(range.start, range.end) {
-                    (range.start, range.end)
-                } else {
-                    (min_signed, max_signed)
-                };
-
-                let niches_ule = niche_start <= niche_end;
-                let niches_sle = sle(niche_start, niche_end);
-                let cast_smaller = cast_to_size <= tag_size;
-
-                // In the algorithm above, we can change
-                // cast(relative_tag) + niche_variants.start()
-                // into
-                // cast(tag + (niche_variants.start() - niche_start))
-                // if either the casted type is no larger than the original
-                // type, or if the niche values are contiguous (in either the
-                // signed or unsigned sense).
-                let can_incr = cast_smaller || niches_ule || niches_sle;
-
-                let data_for_boundary_niche = || -> Option<(IntPredicate, u128)> {
-                    if !can_incr {
-                        None
-                    } else if niche_start == low_unsigned {
-                        Some((IntPredicate::IntULE, niche_end))
-                    } else if niche_end == high_unsigned {
-                        Some((IntPredicate::IntUGE, niche_start))
-                    } else if niche_start == low_signed {
-                        Some((IntPredicate::IntSLE, niche_end))
-                    } else if niche_end == high_signed {
-                        Some((IntPredicate::IntSGE, niche_start))
-                    } else {
-                        None
-                    }
-                };
-
                 let (is_niche, tagged_discr, delta) = if relative_max == 0 {
                     // Best case scenario: only one tagged variant. This will
                     // likely become just a comparison and a jump.
@@ -349,40 +291,6 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                     let tagged_discr =
                         bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64);
                     (is_niche, tagged_discr, 0)
-                } else if let Some((predicate, constant)) = data_for_boundary_niche() {
-                    // The niche values are either the lowest or the highest in
-                    // `range`. We can avoid the first subtraction in the
-                    // algorithm.
-                    // The algorithm is now this:
-                    // is_niche = tag <= niche_end
-                    // discr = if is_niche {
-                    //     cast(tag + (niche_variants.start() - niche_start))
-                    // } else {
-                    //     untagged_variant
-                    // }
-                    // (the first line may instead be tag >= niche_start,
-                    // and may be a signed or unsigned comparison)
-                    // The arithmetic must be done before the cast, so we can
-                    // have the correct wrapping behavior. See issue #104519 for
-                    // the consequences of getting this wrong.
-                    let is_niche =
-                        bx.icmp(predicate, tag, bx.cx().const_uint_big(tag_llty, constant));
-                    let delta = (niche_variants.start().as_u32() as u128).wrapping_sub(niche_start);
-                    let incr_tag = if delta == 0 {
-                        tag
-                    } else {
-                        bx.add(tag, bx.cx().const_uint_big(tag_llty, delta))
-                    };
-
-                    let cast_tag = if cast_smaller {
-                        bx.intcast(incr_tag, cast_to, false)
-                    } else if niches_ule {
-                        bx.zext(incr_tag, cast_to)
-                    } else {
-                        bx.sext(incr_tag, cast_to)
-                    };
-
-                    (is_niche, cast_tag, 0)
                 } else {
                     // The special cases don't apply, so we'll have to go with
                     // the general algorithm.
@@ -558,6 +466,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     bug!("using operand local {:?} as place", place_ref);
                 }
             }
+            LocalRef::PendingOperand => {
+                bug!("using still-pending operand local {:?} as place", place_ref);
+            }
         };
         for elem in place_ref.projection[base..].iter() {
             cg_base = match *elem {
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 3d856986f..d88226f5d 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -10,10 +10,10 @@ use crate::MemFlags;
 use rustc_middle::mir;
 use rustc_middle::mir::Operand;
 use rustc_middle::ty::cast::{CastTy, IntTy};
-use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt};
 use rustc_span::source_map::{Span, DUMMY_SP};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{self, FIRST_VARIANT};
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     #[instrument(level = "trace", skip(self, bx))]
@@ -72,6 +72,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
+            mir::Rvalue::Cast(mir::CastKind::Transmute, ref operand, _ty) => {
+                let src = self.codegen_operand(bx, operand);
+                self.codegen_transmute(bx, src, dest);
+            }
+
             mir::Rvalue::Repeat(ref elem, count) => {
                 let cg_elem = self.codegen_operand(bx, elem);
 
@@ -113,21 +118,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         let variant_dest = dest.project_downcast(bx, variant_index);
                         (variant_index, variant_dest, active_field_index)
                     }
-                    _ => (VariantIdx::from_u32(0), dest, None),
+                    _ => (FIRST_VARIANT, dest, None),
                 };
                 if active_field_index.is_some() {
                     assert_eq!(operands.len(), 1);
                 }
-                for (i, operand) in operands.iter().enumerate() {
+                for (i, operand) in operands.iter_enumerated() {
                     let op = self.codegen_operand(bx, operand);
                     // Do not generate stores and GEPis for zero-sized fields.
                     if !op.layout.is_zst() {
                         let field_index = active_field_index.unwrap_or(i);
                         let field = if let mir::AggregateKind::Array(_) = **kind {
-                            let llindex = bx.cx().const_usize(field_index as u64);
+                            let llindex = bx.cx().const_usize(field_index.as_u32().into());
                             variant_dest.project_index(bx, llindex)
                         } else {
-                            variant_dest.project_field(bx, field_index)
+                            variant_dest.project_field(bx, field_index.as_usize())
                         };
                         op.val.store(bx, field);
                     }
@@ -143,6 +148,156 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         }
     }
 
+    fn codegen_transmute(
+        &mut self,
+        bx: &mut Bx,
+        src: OperandRef<'tcx, Bx::Value>,
+        dst: PlaceRef<'tcx, Bx::Value>,
+    ) {
+        // The MIR validator enforces no unsized transmutes.
+        debug_assert!(src.layout.is_sized());
+        debug_assert!(dst.layout.is_sized());
+
+        if let Some(val) = self.codegen_transmute_operand(bx, src, dst.layout) {
+            val.store(bx, dst);
+            return;
+        }
+
+        match src.val {
+            OperandValue::Ref(..) => {
+                span_bug!(
+                    self.mir.span,
+                    "Operand path should have handled transmute \
+                    from `Ref` {src:?} to place {dst:?}"
+                );
+            }
+            OperandValue::Immediate(..) | OperandValue::Pair(..) => {
+                // When we have immediate(s), the alignment of the source is irrelevant,
+                // so we can store them using the destination's alignment.
+                let llty = bx.backend_type(src.layout);
+                let cast_ptr = bx.pointercast(dst.llval, bx.type_ptr_to(llty));
+                src.val.store(bx, PlaceRef::new_sized_aligned(cast_ptr, src.layout, dst.align));
+            }
+        }
+    }
+
+    /// Attempts to transmute an `OperandValue` to another `OperandValue`.
+    ///
+    /// Returns `None` for cases that can't work in that framework, such as for
+    /// `Immediate`->`Ref` that needs an `alloc` to get the location.
+    fn codegen_transmute_operand(
+        &mut self,
+        bx: &mut Bx,
+        operand: OperandRef<'tcx, Bx::Value>,
+        cast: TyAndLayout<'tcx>,
+    ) -> Option<OperandValue<Bx::Value>> {
+        // Check for transmutes that are always UB.
+        if operand.layout.size != cast.size
+            || operand.layout.abi.is_uninhabited()
+            || cast.abi.is_uninhabited()
+        {
+            if !operand.layout.abi.is_uninhabited() {
+                // Since this is known statically and the input could have existed
+                // without already having hit UB, might as well trap for it.
+                bx.abort();
+            }
+
+            // Because this transmute is UB, return something easy to generate,
+            // since it's fine that later uses of the value are probably UB.
+            return Some(OperandValue::poison(bx, cast));
+        }
+
+        let operand_kind = self.value_kind(operand.layout);
+        let cast_kind = self.value_kind(cast);
+
+        match operand.val {
+            OperandValue::Ref(ptr, meta, align) => {
+                debug_assert_eq!(meta, None);
+                debug_assert!(matches!(operand_kind, OperandValueKind::Ref));
+                let cast_bty = bx.backend_type(cast);
+                let cast_ptr = bx.pointercast(ptr, bx.type_ptr_to(cast_bty));
+                let fake_place = PlaceRef::new_sized_aligned(cast_ptr, cast, align);
+                Some(bx.load_operand(fake_place).val)
+            }
+            OperandValue::Immediate(imm) => {
+                let OperandValueKind::Immediate(in_scalar) = operand_kind else {
+                    bug!("Found {operand_kind:?} for operand {operand:?}");
+                };
+                if let OperandValueKind::Immediate(out_scalar) = cast_kind {
+                    match (in_scalar, out_scalar) {
+                        (ScalarOrZst::Zst, ScalarOrZst::Zst) => {
+                            Some(OperandRef::new_zst(bx, cast).val)
+                        }
+                        (ScalarOrZst::Scalar(in_scalar), ScalarOrZst::Scalar(out_scalar))
+                            if in_scalar.size(self.cx) == out_scalar.size(self.cx) =>
+                        {
+                            let cast_bty = bx.backend_type(cast);
+                            Some(OperandValue::Immediate(
+                                self.transmute_immediate(bx, imm, in_scalar, out_scalar, cast_bty),
+                            ))
+                        }
+                        _ => None,
+                    }
+                } else {
+                    None
+                }
+            }
+            OperandValue::Pair(imm_a, imm_b) => {
+                let OperandValueKind::Pair(in_a, in_b) = operand_kind else {
+                    bug!("Found {operand_kind:?} for operand {operand:?}");
+                };
+                if let OperandValueKind::Pair(out_a, out_b) = cast_kind
+                    && in_a.size(self.cx) == out_a.size(self.cx)
+                    && in_b.size(self.cx) == out_b.size(self.cx)
+                {
+                    let out_a_ibty = bx.scalar_pair_element_backend_type(cast, 0, false);
+                    let out_b_ibty = bx.scalar_pair_element_backend_type(cast, 1, false);
+                    Some(OperandValue::Pair(
+                        self.transmute_immediate(bx, imm_a, in_a, out_a, out_a_ibty),
+                        self.transmute_immediate(bx, imm_b, in_b, out_b, out_b_ibty),
+                    ))
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    /// Transmutes one of the immediates from an [`OperandValue::Immediate`]
+    /// or an [`OperandValue::Pair`] to an immediate of the target type.
+    ///
+    /// `to_backend_ty` must be the *non*-immediate backend type (so it will be
+    /// `i8`, not `i1`, for `bool`-like types.)
+    fn transmute_immediate(
+        &self,
+        bx: &mut Bx,
+        mut imm: Bx::Value,
+        from_scalar: abi::Scalar,
+        to_scalar: abi::Scalar,
+        to_backend_ty: Bx::Type,
+    ) -> Bx::Value {
+        debug_assert_eq!(from_scalar.size(self.cx), to_scalar.size(self.cx));
+
+        use abi::Primitive::*;
+        imm = bx.from_immediate(imm);
+        imm = match (from_scalar.primitive(), to_scalar.primitive()) {
+            (Int(..) | F32 | F64, Int(..) | F32 | F64) => bx.bitcast(imm, to_backend_ty),
+            (Pointer(..), Pointer(..)) => bx.pointercast(imm, to_backend_ty),
+            (Int(..), Pointer(..)) => bx.inttoptr(imm, to_backend_ty),
+            (Pointer(..), Int(..)) => bx.ptrtoint(imm, to_backend_ty),
+            (F32 | F64, Pointer(..)) => {
+                let int_imm = bx.bitcast(imm, bx.cx().type_isize());
+                bx.inttoptr(int_imm, to_backend_ty)
+            }
+            (Pointer(..), F32 | F64) => {
+                let int_imm = bx.ptrtoint(imm, bx.cx().type_isize());
+                bx.bitcast(int_imm, to_backend_ty)
+            }
+        };
+        imm = bx.to_immediate_scalar(imm, to_scalar);
+        imm
+    }
+
     pub fn codegen_rvalue_unsized(
         &mut self,
         bx: &mut Bx,
@@ -295,7 +450,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         assert!(bx.cx().is_backend_immediate(cast));
                         let ll_t_out = bx.cx().immediate_backend_type(cast);
                         if operand.layout.abi.is_uninhabited() {
-                            let val = OperandValue::Immediate(bx.cx().const_undef(ll_t_out));
+                            let val = OperandValue::Immediate(bx.cx().const_poison(ll_t_out));
                             return OperandRef { val, layout: cast };
                         }
                         let r_t_in =
@@ -344,6 +499,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         };
                         OperandValue::Immediate(newval)
                     }
+                    mir::CastKind::Transmute => {
+                        self.codegen_transmute_operand(bx, operand, cast).unwrap_or_else(|| {
+                            bug!("Unsupported transmute-as-operand of {operand:?} to {cast:?}");
+                        })
+                    }
                 };
                 OperandRef { val, layout: cast }
             }
@@ -462,8 +622,20 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             mir::Rvalue::ThreadLocalRef(def_id) => {
                 assert!(bx.cx().tcx().is_static(def_id));
-                let static_ = bx.get_static(def_id);
                 let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id));
+                let static_ = if !def_id.is_local() && bx.cx().tcx().needs_thread_local_shim(def_id)
+                {
+                    let instance = ty::Instance {
+                        def: ty::InstanceDef::ThreadLocalShim(def_id),
+                        substs: ty::InternalSubsts::empty(),
+                    };
+                    let fn_ptr = bx.get_fn_addr(instance);
+                    let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty());
+                    let fn_ty = bx.fn_decl_backend_type(&fn_abi);
+                    bx.call(fn_ty, Some(fn_abi), fn_ptr, &[], None)
+                } else {
+                    bx.get_static(def_id)
+                };
                 OperandRef { val: OperandValue::Immediate(static_), layout }
             }
             mir::Rvalue::Use(ref operand) => self.codegen_operand(bx, operand),
@@ -491,7 +663,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // ZST are passed as operands and require special handling
         // because codegen_place() panics if Local is operand.
         if let Some(index) = place.as_local() {
-            if let LocalRef::Operand(Some(op)) = self.locals[index] {
+            if let LocalRef::Operand(op) = self.locals[index] {
                 if let ty::Array(_, n) = op.layout.ty.kind() {
                     let n = n.eval_target_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
                     return bx.cx().const_usize(n);
@@ -663,17 +835,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 };
                 bx.checked_binop(oop, input_ty, lhs, rhs)
             }
-            mir::BinOp::Shl | mir::BinOp::Shr => {
-                let lhs_llty = bx.cx().val_ty(lhs);
-                let rhs_llty = bx.cx().val_ty(rhs);
-                let invert_mask = common::shift_mask_val(bx, lhs_llty, rhs_llty, true);
-                let outer_bits = bx.and(rhs, invert_mask);
-
-                let of = bx.icmp(IntPredicate::IntNE, outer_bits, bx.cx().const_null(rhs_llty));
-                let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty);
-
-                (val, of)
-            }
             _ => bug!("Operator `{:?}` is not a checkable operator", op),
         };
 
@@ -684,6 +845,31 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool {
         match *rvalue {
+            mir::Rvalue::Cast(mir::CastKind::Transmute, ref operand, cast_ty) => {
+                let operand_ty = operand.ty(self.mir, self.cx.tcx());
+                let cast_layout = self.cx.layout_of(self.monomorphize(cast_ty));
+                let operand_layout = self.cx.layout_of(self.monomorphize(operand_ty));
+
+                match (self.value_kind(operand_layout), self.value_kind(cast_layout)) {
+                    // Can always load from a pointer as needed
+                    (OperandValueKind::Ref, _) => true,
+
+                    // Need to generate an `alloc` to get a pointer from an immediate
+                    (OperandValueKind::Immediate(..) | OperandValueKind::Pair(..), OperandValueKind::Ref) => false,
+
+                    // When we have scalar immediates, we can only convert things
+                    // where the sizes match, to avoid endianness questions.
+                    (OperandValueKind::Immediate(a), OperandValueKind::Immediate(b)) =>
+                        a.size(self.cx) == b.size(self.cx),
+                    (OperandValueKind::Pair(a0, a1), OperandValueKind::Pair(b0, b1)) =>
+                        a0.size(self.cx) == b0.size(self.cx) && a1.size(self.cx) == b1.size(self.cx),
+
+                    // Send mixings between scalars and pairs through the memory route
+                    // FIXME: Maybe this could use insertvalue/extractvalue instead?
+                    (OperandValueKind::Immediate(..), OperandValueKind::Pair(..)) |
+                    (OperandValueKind::Pair(..), OperandValueKind::Immediate(..)) => false,
+                }
+            }
             mir::Rvalue::Ref(..) |
             mir::Rvalue::CopyForDeref(..) |
             mir::Rvalue::AddressOf(..) |
@@ -708,4 +894,52 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         // (*) this is only true if the type is suitable
     }
+
+    /// Gets which variant of [`OperandValue`] is expected for a particular type.
+    fn value_kind(&self, layout: TyAndLayout<'tcx>) -> OperandValueKind {
+        if self.cx.is_backend_immediate(layout) {
+            debug_assert!(!self.cx.is_backend_scalar_pair(layout));
+            OperandValueKind::Immediate(match layout.abi {
+                abi::Abi::Scalar(s) => ScalarOrZst::Scalar(s),
+                abi::Abi::Vector { element, .. } => ScalarOrZst::Scalar(element),
+                _ if layout.is_zst() => ScalarOrZst::Zst,
+                x => span_bug!(self.mir.span, "Couldn't translate {x:?} as backend immediate"),
+            })
+        } else if self.cx.is_backend_scalar_pair(layout) {
+            let abi::Abi::ScalarPair(s1, s2) = layout.abi else {
+                span_bug!(
+                    self.mir.span,
+                    "Couldn't translate {:?} as backend scalar pair",
+                    layout.abi,
+                );
+            };
+            OperandValueKind::Pair(s1, s2)
+        } else {
+            OperandValueKind::Ref
+        }
+    }
+}
+
+/// The variants of this match [`OperandValue`], giving details about the
+/// backend values that will be held in that other type.
+#[derive(Debug, Copy, Clone)]
+enum OperandValueKind {
+    Ref,
+    Immediate(ScalarOrZst),
+    Pair(abi::Scalar, abi::Scalar),
+}
+
+#[derive(Debug, Copy, Clone)]
+enum ScalarOrZst {
+    Zst,
+    Scalar(abi::Scalar),
+}
+
+impl ScalarOrZst {
+    pub fn size(self, cx: &impl abi::HasDataLayout) -> abi::Size {
+        match self {
+            ScalarOrZst::Zst => abi::Size::ZERO,
+            ScalarOrZst::Scalar(s) => s.size(cx),
+        }
+    }
 }
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
index 60fbceb34..3fd7397ad 100644
--- a/compiler/rustc_codegen_ssa/src/mir/statement.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -18,12 +18,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         LocalRef::UnsizedPlace(cg_indirect_dest) => {
                             self.codegen_rvalue_unsized(bx, cg_indirect_dest, rvalue)
                         }
-                        LocalRef::Operand(None) => {
+                        LocalRef::PendingOperand => {
                             let operand = self.codegen_rvalue_operand(bx, rvalue);
-                            self.locals[index] = LocalRef::Operand(Some(operand));
+                            self.locals[index] = LocalRef::Operand(operand);
                             self.debug_introduce_local(bx, index);
                         }
-                        LocalRef::Operand(Some(op)) => {
+                        LocalRef::Operand(op) => {
                             if !op.layout.is_zst() {
                                 span_bug!(
                                     statement.source_info.span,
@@ -92,6 +92,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             | mir::StatementKind::Retag { .. }
             | mir::StatementKind::AscribeUserType(..)
             | mir::StatementKind::ConstEvalCounter
+            | mir::StatementKind::PlaceMention(..)
             | mir::StatementKind::Nop => {}
         }
     }
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index e59fad99a..611dd3d1c 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -1,14 +1,14 @@
 use rustc_ast::ast;
 use rustc_attr::InstructionSetAttr;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::Applicability;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{DefIdTree, TyCtxt};
+use rustc_middle::ty::TyCtxt;
 use rustc_session::parse::feature_err;
 use rustc_session::Session;
 use rustc_span::symbol::sym;
@@ -192,7 +192,7 @@ const X86_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("fxsr", None),
     ("gfni", Some(sym::avx512_target_feature)),
     ("lzcnt", None),
-    ("movbe", Some(sym::movbe_target_feature)),
+    ("movbe", None),
     ("pclmulqdq", None),
     ("popcnt", None),
     ("rdrand", None),
@@ -251,6 +251,7 @@ const RISCV_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("e", Some(sym::riscv_target_feature)),
     ("f", Some(sym::riscv_target_feature)),
     ("m", Some(sym::riscv_target_feature)),
+    ("relax", Some(sym::riscv_target_feature)),
     ("v", Some(sym::riscv_target_feature)),
     ("zba", Some(sym::riscv_target_feature)),
     ("zbb", Some(sym::riscv_target_feature)),
@@ -394,7 +395,6 @@ pub fn from_target_feature(
                 Some(sym::sse4a_target_feature) => rust_features.sse4a_target_feature,
                 Some(sym::tbm_target_feature) => rust_features.tbm_target_feature,
                 Some(sym::wasm_target_feature) => rust_features.wasm_target_feature,
-                Some(sym::movbe_target_feature) => rust_features.movbe_target_feature,
                 Some(sym::rtm_target_feature) => rust_features.rtm_target_feature,
                 Some(sym::ermsb_target_feature) => rust_features.ermsb_target_feature,
                 Some(sym::bpf_target_feature) => rust_features.bpf_target_feature,
@@ -418,7 +418,7 @@ pub fn from_target_feature(
 
 /// Computes the set of target features used in a function for the purposes of
 /// inline assembly.
-fn asm_target_features(tcx: TyCtxt<'_>, did: DefId) -> &FxHashSet<Symbol> {
+fn asm_target_features(tcx: TyCtxt<'_>, did: DefId) -> &FxIndexSet<Symbol> {
     let mut target_features = tcx.sess.unstable_target_features.clone();
     if tcx.def_kind(did).has_codegen_attrs() {
         let attrs = tcx.codegen_fn_attrs(did);
@@ -442,7 +442,7 @@ fn asm_target_features(tcx: TyCtxt<'_>, did: DefId) -> &FxHashSet<Symbol> {
 pub fn check_target_feature_trait_unsafe(tcx: TyCtxt<'_>, id: LocalDefId, attr_span: Span) {
     if let DefKind::AssocFn = tcx.def_kind(id) {
         let parent_id = tcx.local_parent(id);
-        if let DefKind::Impl { of_trait: true } = tcx.def_kind(parent_id) {
+        if let DefKind::Trait | DefKind::Impl { of_trait: true } = tcx.def_kind(parent_id) {
             tcx.sess
                 .struct_span_err(
                     attr_span,
diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs
index fdc7a30e8..619063027 100644
--- a/compiler/rustc_codegen_ssa/src/traits/consts.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs
@@ -8,6 +8,7 @@ pub trait ConstMethods<'tcx>: BackendTypes {
     // Constant constructors
     fn const_null(&self, t: Self::Type) -> Self::Value;
     fn const_undef(&self, t: Self::Type) -> Self::Value;
+    fn const_poison(&self, t: Self::Type) -> Self::Value;
     fn const_int(&self, t: Self::Type, i: i64) -> Self::Value;
     fn const_uint(&self, t: Self::Type, i: u64) -> Self::Value;
     fn const_uint_big(&self, t: Self::Type, u: u128) -> Self::Value;
diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs
index 109161ccc..32905b079 100644
--- a/compiler/rustc_codegen_ssa/src/traits/type_.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs
@@ -100,11 +100,22 @@ pub trait DerivedTypeMethods<'tcx>: BaseTypeMethods<'tcx> + MiscMethods<'tcx> {
 impl<'tcx, T> DerivedTypeMethods<'tcx> for T where Self: BaseTypeMethods<'tcx> + MiscMethods<'tcx> {}
 
 pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> {
+    /// The backend type used for a rust type when it's in memory,
+    /// such as when it's stack-allocated or when it's being loaded or stored.
     fn backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type;
     fn cast_backend_type(&self, ty: &CastTarget) -> Self::Type;
     fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type;
     fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type;
     fn reg_backend_type(&self, ty: &Reg) -> Self::Type;
+    /// The backend type used for a rust type when it's in an SSA register.
+    ///
+    /// For nearly all types this is the same as the [`Self::backend_type`], however
+    /// `bool` (and other `0`-or-`1` values) are kept as [`BaseTypeMethods::type_i1`]
+    /// in registers but as [`BaseTypeMethods::type_i8`] in memory.
+    ///
+    /// Converting values between the two different backend types is done using
+    /// [`from_immediate`](super::BuilderMethods::from_immediate) and
+    /// [`to_immediate_scalar`](super::BuilderMethods::to_immediate_scalar).
     fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type;
     fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool;
     fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool;
diff --git a/compiler/rustc_const_eval/locales/en-US.ftl b/compiler/rustc_const_eval/messages.ftl
index 33bb116d6..f6751df44 100644
--- a/compiler/rustc_const_eval/locales/en-US.ftl
+++ b/compiler/rustc_const_eval/messages.ftl
@@ -39,17 +39,25 @@ const_eval_unstable_const_fn = `{$def_path}` is not yet stable as a const fn
 const_eval_unallowed_mutable_refs =
     mutable references are not allowed in the final value of {$kind}s
     .teach_note =
-        References in statics and constants may only refer to immutable values.\n\n
+        References in statics and constants may only refer to immutable values.
+
+
         Statics are shared everywhere, and if they refer to mutable data one might violate memory
-        safety since holding multiple mutable references to shared data is not allowed.\n\n
+        safety since holding multiple mutable references to shared data is not allowed.
+
+
         If you really want global mutable state, try using static mut or a global UnsafeCell.
 
 const_eval_unallowed_mutable_refs_raw =
     raw mutable references are not allowed in the final value of {$kind}s
     .teach_note =
-        References in statics and constants may only refer to immutable values.\n\n
+        References in statics and constants may only refer to immutable values.
+
+
         Statics are shared everywhere, and if they refer to mutable data one might violate memory
-        safety since holding multiple mutable references to shared data is not allowed.\n\n
+        safety since holding multiple mutable references to shared data is not allowed.
+
+
         If you really want global mutable state, try using static mut or a global UnsafeCell.
 
 const_eval_non_const_fmt_macro_call =
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index 7564ba17b..4bd6fe199 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -205,7 +205,7 @@ pub(crate) fn turn_into_const_value<'tcx>(
     let cid = key.value;
     let def_id = cid.instance.def.def_id();
     let is_static = tcx.is_static(def_id);
-    // This is just accessing an already computed constant, so no need to check alginment here.
+    // This is just accessing an already computed constant, so no need to check alignment here.
     let ecx = mk_eval_cx(
         tcx,
         tcx.def_span(key.value.instance.def_id()),
diff --git a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
index 9eaab1f47..088a824fd 100644
--- a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
@@ -1,15 +1,22 @@
+use rustc_attr as attr;
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{DefIdTree, TyCtxt};
+use rustc_middle::ty::TyCtxt;
 use rustc_span::symbol::Symbol;
 
-/// Whether the `def_id` is an unstable const fn and what feature gate is necessary to enable it
-pub fn is_unstable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> Option<Symbol> {
+/// Whether the `def_id` is an unstable const fn and what feature gate(s) are necessary to enable
+/// it.
+pub fn is_unstable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> Option<(Symbol, Option<Symbol>)> {
     if tcx.is_const_fn_raw(def_id) {
         let const_stab = tcx.lookup_const_stability(def_id)?;
-        if const_stab.is_const_unstable() { Some(const_stab.feature) } else { None }
+        match const_stab.level {
+            attr::StabilityLevel::Unstable { implied_by, .. } => {
+                Some((const_stab.feature, implied_by))
+            }
+            attr::StabilityLevel::Stable { .. } => None,
+        }
     } else {
         None
     }
@@ -25,8 +32,7 @@ pub fn is_parent_const_impl_raw(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
 /// it is a trait impl/function, return if it has a `const` modifier. If it is an intrinsic,
 /// report whether said intrinsic has a `rustc_const_{un,}stable` attribute. Otherwise, return
 /// `Constness::NotConst`.
-fn constness(tcx: TyCtxt<'_>, def_id: DefId) -> hir::Constness {
-    let def_id = def_id.expect_local();
+fn constness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Constness {
     let node = tcx.hir().get_by_def_id(def_id);
 
     match node {
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index a44f70ed0..a5dfd1072 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -2,7 +2,7 @@ use rustc_hir::def::DefKind;
 use rustc_hir::{LangItem, CRATE_HIR_ID};
 use rustc_middle::mir;
 use rustc_middle::mir::interpret::PointerArithmetic;
-use rustc_middle::ty::layout::FnAbiOf;
+use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_session::lint::builtin::INVALID_ALIGNMENT;
 use std::borrow::Borrow;
@@ -23,7 +23,7 @@ use rustc_target::spec::abi::Abi as CallAbi;
 
 use crate::interpret::{
     self, compile_time_machine, AllocId, ConstAllocation, FnVal, Frame, ImmTy, InterpCx,
-    InterpResult, OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind,
+    InterpResult, OpTy, PlaceTy, Pointer, Scalar,
 };
 
 use super::error::*;
@@ -271,7 +271,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
                         /* with_caller_location = */ false,
                         dest,
                         ret,
-                        StackPopUnwind::NotAllowed,
+                        mir::UnwindAction::Unreachable,
                     )?;
                     Ok(ControlFlow::Break(()))
                 } else {
@@ -335,8 +335,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
     }
 
     #[inline(always)]
-    fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool {
-        ecx.tcx.sess.opts.unstable_opts.extra_const_ub_checks
+    fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>, layout: TyAndLayout<'tcx>) -> bool {
+        ecx.tcx.sess.opts.unstable_opts.extra_const_ub_checks || layout.abi.is_uninhabited()
     }
 
     fn alignment_check_failed(
@@ -401,7 +401,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         args: &[OpTy<'tcx>],
         dest: &PlaceTy<'tcx>,
         ret: Option<mir::BasicBlock>,
-        _unwind: StackPopUnwind, // unwinding is not supported in consts
+        _unwind: mir::UnwindAction, // unwinding is not supported in consts
     ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> {
         debug!("find_mir_or_eval_fn: {:?}", instance);
 
@@ -450,7 +450,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         args: &[OpTy<'tcx>],
         dest: &PlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
-        _unwind: StackPopUnwind,
+        _unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx> {
         // Shared intrinsics.
         if ecx.emulate_intrinsic(instance, args, dest, target)? {
@@ -526,7 +526,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
     fn assert_panic(
         ecx: &mut InterpCx<'mir, 'tcx, Self>,
         msg: &AssertMessage<'tcx>,
-        _unwind: Option<mir::BasicBlock>,
+        _unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx> {
         use rustc_middle::mir::AssertKind::*;
         // Convert `AssertKind<Operand>` to `AssertKind<Scalar>`.
@@ -544,6 +544,12 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
             RemainderByZero(op) => RemainderByZero(eval_to_int(op)?),
             ResumedAfterReturn(generator_kind) => ResumedAfterReturn(*generator_kind),
             ResumedAfterPanic(generator_kind) => ResumedAfterPanic(*generator_kind),
+            MisalignedPointerDereference { ref required, ref found } => {
+                MisalignedPointerDereference {
+                    required: eval_to_int(required)?,
+                    found: eval_to_int(found)?,
+                }
+            }
         };
         Err(ConstEvalErrKind::AssertFailure(err).into())
     }
diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
index a73f778d4..4d54c0183 100644
--- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs
+++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
@@ -8,7 +8,7 @@ use crate::interpret::{
 use crate::interpret::{MPlaceTy, Value};
 use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt};
 use rustc_span::source_map::DUMMY_SP;
-use rustc_target::abi::{Align, VariantIdx};
+use rustc_target::abi::{Align, FieldIdx, VariantIdx, FIRST_VARIANT};
 
 #[instrument(skip(ecx), level = "debug")]
 fn branches<'tcx>(
@@ -412,7 +412,8 @@ fn valtree_into_mplace<'tcx>(
 
                         let inner_ty = match ty.kind() {
                             ty::Adt(def, substs) => {
-                                def.variant(VariantIdx::from_u32(0)).fields[i].ty(tcx, substs)
+                                let i = FieldIdx::from_usize(i);
+                                def.variant(FIRST_VARIANT).fields[i].ty(tcx, substs)
                             }
                             ty::Tuple(inner_tys) => inner_tys[i],
                             _ => bug!("unexpected unsized type {:?}", ty),
diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs
index 2be5ed896..163e3f869 100644
--- a/compiler/rustc_const_eval/src/interpret/cast.rs
+++ b/compiler/rustc_const_eval/src/interpret/cast.rs
@@ -67,12 +67,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
 
             Pointer(PointerCast::ReifyFnPointer) => {
+                // All reifications must be monomorphic, bail out otherwise.
+                ensure_monomorphic_enough(*self.tcx, src.layout.ty)?;
+
                 // The src operand does not matter, just its type
                 match *src.layout.ty.kind() {
                     ty::FnDef(def_id, substs) => {
-                        // All reifications must be monomorphic, bail out otherwise.
-                        ensure_monomorphic_enough(*self.tcx, src.layout.ty)?;
-
                         let instance = ty::Instance::resolve_for_fn_ptr(
                             *self.tcx,
                             self.param_env,
@@ -100,12 +100,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
 
             Pointer(PointerCast::ClosureFnPointer(_)) => {
+                // All reifications must be monomorphic, bail out otherwise.
+                ensure_monomorphic_enough(*self.tcx, src.layout.ty)?;
+
                 // The src operand does not matter, just its type
                 match *src.layout.ty.kind() {
                     ty::Closure(def_id, substs) => {
-                        // All reifications must be monomorphic, bail out otherwise.
-                        ensure_monomorphic_enough(*self.tcx, src.layout.ty)?;
-
                         let instance = ty::Instance::resolve_closure(
                             *self.tcx,
                             def_id,
@@ -133,6 +133,22 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     bug!()
                 }
             }
+
+            Transmute => {
+                assert!(src.layout.is_sized());
+                assert!(dest.layout.is_sized());
+                if src.layout.size != dest.layout.size {
+                    throw_ub_format!(
+                        "transmuting from {}-byte type to {}-byte type: `{}` -> `{}`",
+                        src.layout.size.bytes(),
+                        dest.layout.size.bytes(),
+                        src.layout.ty,
+                        dest.layout.ty,
+                    );
+                }
+
+                self.copy_op(src, dest, /*allow_transmute*/ true)?;
+            }
         }
         Ok(())
     }
@@ -359,8 +375,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let val = Immediate::new_dyn_trait(ptr, vtable, &*self.tcx);
                 self.write_immediate(val, dest)
             }
-
             _ => {
+                // Do not ICE if we are not monomorphic enough.
+                ensure_monomorphic_enough(*self.tcx, src.layout.ty)?;
+                ensure_monomorphic_enough(*self.tcx, cast_ty)?;
+
                 span_bug!(
                     self.cur_span(),
                     "invalid pointer unsizing {:?} -> {:?}",
@@ -404,12 +423,18 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 }
                 Ok(())
             }
-            _ => span_bug!(
-                self.cur_span(),
-                "unsize_into: invalid conversion: {:?} -> {:?}",
-                src.layout,
-                dest.layout
-            ),
+            _ => {
+                // Do not ICE if we are not monomorphic enough.
+                ensure_monomorphic_enough(*self.tcx, src.layout.ty)?;
+                ensure_monomorphic_enough(*self.tcx, cast_ty.ty)?;
+
+                span_bug!(
+                    self.cur_span(),
+                    "unsize_into: invalid conversion: {:?} -> {:?}",
+                    src.layout,
+                    dest.layout
+                )
+            }
         }
     }
 }
diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs
index 3db102e48..3e58a58ae 100644
--- a/compiler/rustc_const_eval/src/interpret/eval_context.rs
+++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs
@@ -7,7 +7,7 @@ use either::{Either, Left, Right};
 use rustc_hir::{self as hir, def_id::DefId, definitions::DefPathData};
 use rustc_index::vec::IndexVec;
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::{ErrorHandled, InterpError, InvalidProgramInfo};
+use rustc_middle::mir::interpret::{ErrorHandled, InterpError};
 use rustc_middle::ty::layout::{
     self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOf, LayoutOfHelpers,
     TyAndLayout,
@@ -139,17 +139,6 @@ pub struct FrameInfo<'tcx> {
     pub lint_root: Option<hir::HirId>,
 }
 
-/// Unwind information.
-#[derive(Clone, Copy, Eq, PartialEq, Debug)]
-pub enum StackPopUnwind {
-    /// The cleanup block.
-    Cleanup(mir::BasicBlock),
-    /// No cleanup needs to be done.
-    Skip,
-    /// Unwinding is not allowed (UB).
-    NotAllowed,
-}
-
 #[derive(Clone, Copy, Eq, PartialEq, Debug)] // Miri debug-prints these
 pub enum StackPopCleanup {
     /// Jump to the next block in the caller, or cause UB if None (that's a function
@@ -157,7 +146,7 @@ pub enum StackPopCleanup {
     /// we can validate it at that layout.
     /// `ret` stores the block we jump to on a normal return, while `unwind`
     /// stores the block used for cleanup during unwinding.
-    Goto { ret: Option<mir::BasicBlock>, unwind: StackPopUnwind },
+    Goto { ret: Option<mir::BasicBlock>, unwind: mir::UnwindAction },
     /// The root frame of the stack: nowhere else to jump to.
     /// `cleanup` says whether locals are deallocated. Static computation
     /// wants them leaked to intern what they need (and just throw away
@@ -508,14 +497,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         frame
             .instance
             .try_subst_mir_and_normalize_erasing_regions(*self.tcx, self.param_env, value)
-            .map_err(|e| {
-                self.tcx.sess.delay_span_bug(
-                    self.cur_span(),
-                    format!("failed to normalize {}", e.get_type_for_failure()).as_str(),
-                );
-
-                InterpError::InvalidProgram(InvalidProgramInfo::TooGeneric)
-            })
+            .map_err(|_| err_inval!(TooGeneric))
     }
 
     /// The `substs` are assumed to already be in our interpreter "universe" (param_env).
@@ -543,24 +525,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         local: mir::Local,
         layout: Option<TyAndLayout<'tcx>>,
     ) -> InterpResult<'tcx, TyAndLayout<'tcx>> {
-        // `const_prop` runs into this with an invalid (empty) frame, so we
-        // have to support that case (mostly by skipping all caching).
-        match frame.locals.get(local).and_then(|state| state.layout.get()) {
-            None => {
-                let layout = from_known_layout(self.tcx, self.param_env, layout, || {
-                    let local_ty = frame.body.local_decls[local].ty;
-                    let local_ty =
-                        self.subst_from_frame_and_normalize_erasing_regions(frame, local_ty)?;
-                    self.layout_of(local_ty)
-                })?;
-                if let Some(state) = frame.locals.get(local) {
-                    // Layouts of locals are requested a lot, so we cache them.
-                    state.layout.set(Some(layout));
-                }
-                Ok(layout)
-            }
-            Some(layout) => Ok(layout),
+        let state = &frame.locals[local];
+        if let Some(layout) = state.layout.get() {
+            return Ok(layout);
         }
+
+        let layout = from_known_layout(self.tcx, self.param_env, layout, || {
+            let local_ty = frame.body.local_decls[local].ty;
+            let local_ty = self.subst_from_frame_and_normalize_erasing_regions(frame, local_ty)?;
+            self.layout_of(local_ty)
+        })?;
+
+        // Layouts of locals are requested a lot, so we cache them.
+        state.layout.set(Some(layout));
+        Ok(layout)
     }
 
     /// Returns the actual dynamic size and alignment of the place at the given type.
@@ -746,18 +724,22 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     /// *Unwind* to the given `target` basic block.
     /// Do *not* use for returning! Use `return_to_block` instead.
     ///
-    /// If `target` is `StackPopUnwind::Skip`, that indicates the function does not need cleanup
+    /// If `target` is `UnwindAction::Continue`, that indicates the function does not need cleanup
     /// during unwinding, and we will just keep propagating that upwards.
     ///
-    /// If `target` is `StackPopUnwind::NotAllowed`, that indicates the function does not allow
+    /// If `target` is `UnwindAction::Unreachable`, that indicates the function does not allow
     /// unwinding, and doing so is UB.
-    pub fn unwind_to_block(&mut self, target: StackPopUnwind) -> InterpResult<'tcx> {
+    pub fn unwind_to_block(&mut self, target: mir::UnwindAction) -> InterpResult<'tcx> {
         self.frame_mut().loc = match target {
-            StackPopUnwind::Cleanup(block) => Left(mir::Location { block, statement_index: 0 }),
-            StackPopUnwind::Skip => Right(self.frame_mut().body.span),
-            StackPopUnwind::NotAllowed => {
+            mir::UnwindAction::Cleanup(block) => Left(mir::Location { block, statement_index: 0 }),
+            mir::UnwindAction::Continue => Right(self.frame_mut().body.span),
+            mir::UnwindAction::Unreachable => {
                 throw_ub_format!("unwinding past a stack frame that does not allow unwinding")
             }
+            mir::UnwindAction::Terminate => {
+                self.frame_mut().loc = Right(self.frame_mut().body.span);
+                M::abort(self, "panic in a function that cannot unwind".to_owned())?;
+            }
         };
         Ok(())
     }
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index a29cdade0..26fb041b4 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -127,7 +127,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // First handle intrinsics without return place.
         let ret = match ret {
             None => match intrinsic_name {
-                sym::transmute => throw_ub_format!("transmuting to uninhabited type"),
                 sym::abort => M::abort(self, "the program aborted execution".to_owned())?,
                 // Unsupported diverging intrinsic.
                 _ => return Ok(false),
@@ -411,9 +410,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 self.exact_div(&val, &size, dest)?;
             }
 
-            sym::transmute => {
-                self.copy_op(&args[0], dest, /*allow_transmute*/ true)?;
-            }
             sym::assert_inhabited
             | sym::assert_zero_valid
             | sym::assert_mem_uninitialized_valid => {
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
index cf52299b7..3701eb93e 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
@@ -77,7 +77,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         line: u32,
         col: u32,
     ) -> MPlaceTy<'tcx, M::Provenance> {
-        let loc_details = &self.tcx.sess.opts.unstable_opts.location_detail;
+        let loc_details = self.tcx.sess.opts.unstable_opts.location_detail;
         // This can fail if rustc runs out of memory right here. Trying to emit an error would be
         // pointless, since that would require allocating more memory than these short strings.
         let file = if loc_details.file {
@@ -111,7 +111,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         location
     }
 
-    pub(crate) fn location_triple_for_span(&self, span: Span) -> (Symbol, u32, u32) {
+    pub(crate) fn location_triple_for_span(&self, mut span: Span) -> (Symbol, u32, u32) {
+        // Remove `Inlined` marks as they pollute `expansion_cause`.
+        while span.is_inlined() {
+            span.remove_mark();
+        }
         let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
         let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo());
         (
diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs
index 92fa59aec..0291cca73 100644
--- a/compiler/rustc_const_eval/src/interpret/machine.rs
+++ b/compiler/rustc_const_eval/src/interpret/machine.rs
@@ -8,6 +8,7 @@ use std::hash::Hash;
 
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_middle::mir;
+use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::def_id::DefId;
 use rustc_target::abi::{Align, Size};
@@ -17,7 +18,7 @@ use crate::const_eval::CheckAlignment;
 
 use super::{
     AllocBytes, AllocId, AllocRange, Allocation, ConstAllocation, Frame, ImmTy, InterpCx,
-    InterpResult, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, StackPopUnwind,
+    InterpResult, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar,
 };
 
 /// Data returned by Machine::stack_pop,
@@ -145,8 +146,8 @@ pub trait Machine<'mir, 'tcx>: Sized {
         check: CheckAlignment,
     ) -> InterpResult<'tcx, ()>;
 
-    /// Whether to enforce the validity invariant
-    fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
+    /// Whether to enforce the validity invariant for a specific layout.
+    fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>, layout: TyAndLayout<'tcx>) -> bool;
 
     /// Whether function calls should be [ABI](CallAbi)-checked.
     fn enforce_abi(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool {
@@ -155,7 +156,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
 
     /// Whether Assert(OverflowNeg) and Assert(Overflow) MIR terminators should actually
     /// check for overflow.
-    fn ignore_checkable_overflow_assertions(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
+    fn ignore_optional_overflow_checks(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
 
     /// Entry point for obtaining the MIR of anything that should get evaluated.
     /// So not just functions and shims, but also const/static initializers, anonymous
@@ -184,7 +185,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
         args: &[OpTy<'tcx, Self::Provenance>],
         destination: &PlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
-        unwind: StackPopUnwind,
+        unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>>;
 
     /// Execute `fn_val`. It is the hook's responsibility to advance the instruction
@@ -196,7 +197,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
         args: &[OpTy<'tcx, Self::Provenance>],
         destination: &PlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
-        unwind: StackPopUnwind,
+        unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx>;
 
     /// Directly process an intrinsic without pushing a stack frame. It is the hook's
@@ -207,17 +208,17 @@ pub trait Machine<'mir, 'tcx>: Sized {
         args: &[OpTy<'tcx, Self::Provenance>],
         destination: &PlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
-        unwind: StackPopUnwind,
+        unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx>;
 
     /// Called to evaluate `Assert` MIR terminators that trigger a panic.
     fn assert_panic(
         ecx: &mut InterpCx<'mir, 'tcx, Self>,
         msg: &mir::AssertMessage<'tcx>,
-        unwind: Option<mir::BasicBlock>,
+        unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx>;
 
-    /// Called to evaluate `Abort` MIR terminator.
+    /// Called to abort evaluation.
     fn abort(_ecx: &mut InterpCx<'mir, 'tcx, Self>, _msg: String) -> InterpResult<'tcx, !> {
         throw_unsup_format!("aborting execution is not supported")
     }
@@ -474,7 +475,7 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
     }
 
     #[inline(always)]
-    fn ignore_checkable_overflow_assertions(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
+    fn ignore_optional_overflow_checks(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
         false
     }
 
@@ -486,7 +487,7 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
         _args: &[OpTy<$tcx>],
         _destination: &PlaceTy<$tcx, Self::Provenance>,
         _target: Option<mir::BasicBlock>,
-        _unwind: StackPopUnwind,
+        _unwind: mir::UnwindAction,
     ) -> InterpResult<$tcx> {
         match fn_val {}
     }
diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs
index 86de4e4e3..898d62361 100644
--- a/compiler/rustc_const_eval/src/interpret/mod.rs
+++ b/compiler/rustc_const_eval/src/interpret/mod.rs
@@ -20,9 +20,7 @@ mod visitor;
 
 pub use rustc_middle::mir::interpret::*; // have all the `interpret` symbols in one place: here
 
-pub use self::eval_context::{
-    Frame, FrameInfo, InterpCx, LocalState, LocalValue, StackPopCleanup, StackPopUnwind,
-};
+pub use self::eval_context::{Frame, FrameInfo, InterpCx, LocalState, LocalValue, StackPopCleanup};
 pub use self::intern::{intern_const_alloc_recursive, InternKind};
 pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackPopJump};
 pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind};
diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs
index 3c463500a..03b09cf83 100644
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@@ -5,10 +5,11 @@
 use either::{Either, Left, Right};
 
 use rustc_ast::Mutability;
+use rustc_index::vec::IndexSlice;
 use rustc_middle::mir;
 use rustc_middle::ty;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
-use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size, VariantIdx};
+use rustc_target::abi::{self, Abi, Align, FieldIdx, HasDataLayout, Size, FIRST_VARIANT};
 
 use super::{
     alloc_range, mir_assign_valid_types, AllocId, AllocRef, AllocRefMut, CheckInAllocMsg,
@@ -461,7 +462,7 @@ where
     ) -> InterpResult<'tcx> {
         self.write_immediate_no_validate(src, dest)?;
 
-        if M::enforce_validity(self) {
+        if M::enforce_validity(self, dest.layout) {
             // Data got changed, better make sure it matches the type!
             self.validate_operand(&self.place_to_op(dest)?)?;
         }
@@ -616,7 +617,7 @@ where
     ) -> InterpResult<'tcx> {
         self.copy_op_no_validate(src, dest, allow_transmute)?;
 
-        if M::enforce_validity(self) {
+        if M::enforce_validity(self, dest.layout) {
             // Data got changed, better make sure it matches the type!
             self.validate_operand(&self.place_to_op(dest)?)?;
         }
@@ -787,7 +788,7 @@ where
     pub fn write_aggregate(
         &mut self,
         kind: &mir::AggregateKind<'tcx>,
-        operands: &[mir::Operand<'tcx>],
+        operands: &IndexSlice<FieldIdx, mir::Operand<'tcx>>,
         dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
         self.write_uninit(&dest)?;
@@ -796,14 +797,14 @@ where
                 let variant_dest = self.place_downcast(&dest, variant_index)?;
                 (variant_index, variant_dest, active_field_index)
             }
-            _ => (VariantIdx::from_u32(0), dest.clone(), None),
+            _ => (FIRST_VARIANT, dest.clone(), None),
         };
         if active_field_index.is_some() {
             assert_eq!(operands.len(), 1);
         }
-        for (field_index, operand) in operands.iter().enumerate() {
+        for (field_index, operand) in operands.iter_enumerated() {
             let field_index = active_field_index.unwrap_or(field_index);
-            let field_dest = self.place_field(&variant_dest, field_index)?;
+            let field_dest = self.place_field(&variant_dest, field_index.as_usize())?;
             let op = self.eval_operand(operand, Some(field_dest.layout))?;
             self.copy_op(&op, &field_dest, /*allow_transmute*/ false)?;
         }
diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs
index 6863435e5..9a366364e 100644
--- a/compiler/rustc_const_eval/src/interpret/step.rs
+++ b/compiler/rustc_const_eval/src/interpret/step.rs
@@ -114,7 +114,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             Intrinsic(box intrinsic) => self.emulate_nondiverging_intrinsic(intrinsic)?,
 
             // Statements we do not track.
-            AscribeUserType(..) => {}
+            PlaceMention(..) | AscribeUserType(..) => {}
 
             // Currently, Miri discards Coverage statements. Coverage statements are only injected
             // via an optional compile time MIR pass and have no side effects. Since Coverage
diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs
index 2aea7c79b..a07702f7d 100644
--- a/compiler/rustc_const_eval/src/interpret/terminator.rs
+++ b/compiler/rustc_const_eval/src/interpret/terminator.rs
@@ -13,7 +13,7 @@ use rustc_target::spec::abi::Abi;
 
 use super::{
     FnVal, ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemoryKind, OpTy, Operand,
-    PlaceTy, Scalar, StackPopCleanup, StackPopUnwind,
+    PlaceTy, Scalar, StackPopCleanup,
 };
 
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
@@ -60,7 +60,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 ref args,
                 destination,
                 target,
-                ref cleanup,
+                unwind,
                 from_hir_call: _,
                 fn_span: _,
             } => {
@@ -106,11 +106,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     with_caller_location,
                     &destination,
                     target,
-                    match (cleanup, fn_abi.can_unwind) {
-                        (Some(cleanup), true) => StackPopUnwind::Cleanup(*cleanup),
-                        (None, true) => StackPopUnwind::Skip,
-                        (_, false) => StackPopUnwind::NotAllowed,
-                    },
+                    if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable },
                 )?;
                 // Sanity-check that `eval_fn_call` either pushed a new frame or
                 // did a jump to another block.
@@ -137,23 +133,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 self.drop_in_place(&place, instance, target, unwind)?;
             }
 
-            Assert { ref cond, expected, ref msg, target, cleanup } => {
-                let ignored = M::ignore_checkable_overflow_assertions(self)
-                    && match msg {
-                        mir::AssertKind::OverflowNeg(..) => true,
-                        mir::AssertKind::Overflow(op, ..) => op.is_checkable(),
-                        _ => false,
-                    };
+            Assert { ref cond, expected, ref msg, target, unwind } => {
+                let ignored =
+                    M::ignore_optional_overflow_checks(self) && msg.is_optional_overflow_check();
                 let cond_val = self.read_scalar(&self.eval_operand(cond, None)?)?.to_bool()?;
                 if ignored || expected == cond_val {
                     self.go_to_block(target);
                 } else {
-                    M::assert_panic(self, msg, cleanup)?;
+                    M::assert_panic(self, msg, unwind)?;
                 }
             }
 
-            Abort => {
-                M::abort(self, "the program aborted execution".to_owned())?;
+            Terminate => {
+                // FIXME: maybe should call `panic_no_unwind` lang item instead.
+                M::abort(self, "panic in a function that cannot unwind".to_owned())?;
             }
 
             // When we encounter Resume, we've finished unwinding
@@ -171,11 +164,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             Unreachable => throw_ub!(Unreachable),
 
             // These should never occur for MIR we actually run.
-            DropAndReplace { .. }
-            | FalseEdge { .. }
-            | FalseUnwind { .. }
-            | Yield { .. }
-            | GeneratorDrop => span_bug!(
+            FalseEdge { .. } | FalseUnwind { .. } | Yield { .. } | GeneratorDrop => span_bug!(
                 terminator.source_info.span,
                 "{:#?} should have been eliminated by MIR pass",
                 terminator.kind
@@ -359,7 +348,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         with_caller_location: bool,
         destination: &PlaceTy<'tcx, M::Provenance>,
         target: Option<mir::BasicBlock>,
-        mut unwind: StackPopUnwind,
+        mut unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx> {
         trace!("eval_fn_call: {:#?}", fn_val);
 
@@ -390,6 +379,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             | ty::InstanceDef::FnPtrShim(..)
             | ty::InstanceDef::DropGlue(..)
             | ty::InstanceDef::CloneShim(..)
+            | ty::InstanceDef::FnPtrAddrShim(..)
+            | ty::InstanceDef::ThreadLocalShim(..)
             | ty::InstanceDef::Item(_) => {
                 // We need MIR for this fn
                 let Some((body, instance)) =
@@ -416,9 +407,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     }
                 }
 
-                if !matches!(unwind, StackPopUnwind::NotAllowed) && !callee_fn_abi.can_unwind {
-                    // The callee cannot unwind.
-                    unwind = StackPopUnwind::NotAllowed;
+                if !callee_fn_abi.can_unwind {
+                    // The callee cannot unwind, so force the `Unreachable` unwind handling.
+                    unwind = mir::UnwindAction::Unreachable;
                 }
 
                 self.push_stack_frame(
@@ -547,7 +538,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let mut receiver = args[0].clone();
                 let receiver_place = loop {
                     match receiver.layout.ty.kind() {
-                        ty::Ref(..) | ty::RawPtr(..) => break self.deref_operand(&receiver)?,
+                        ty::Ref(..) | ty::RawPtr(..) => {
+                            // We do *not* use `deref_operand` here: we don't want to conceptually
+                            // create a place that must be dereferenceable, since the receiver might
+                            // be a raw pointer and (for `*const dyn Trait`) we don't need to
+                            // actually access memory to resolve this method.
+                            // Also see <https://github.com/rust-lang/miri/issues/2786>.
+                            let val = self.read_immediate(&receiver)?;
+                            break self.ref_to_mplace(&val)?;
+                        }
                         ty::Dynamic(.., ty::Dyn) => break receiver.assert_mem_place(), // no immediate unsized values
                         ty::Dynamic(.., ty::DynStar) => {
                             // Not clear how to handle this, so far we assume the receiver is always a pointer.
@@ -674,7 +673,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         place: &PlaceTy<'tcx, M::Provenance>,
         instance: ty::Instance<'tcx>,
         target: mir::BasicBlock,
-        unwind: Option<mir::BasicBlock>,
+        unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx> {
         trace!("drop_in_place: {:?},\n  {:?}, {:?}", *place, place.layout.ty, instance);
         // We take the address of the object. This may well be unaligned, which is fine
@@ -715,10 +714,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             false,
             &ret.into(),
             Some(target),
-            match unwind {
-                Some(cleanup) => StackPopUnwind::Cleanup(cleanup),
-                None => StackPopUnwind::Skip,
-            },
+            unwind,
         )
     }
 }
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index f7881c509..93b5273e1 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -16,7 +16,9 @@ use rustc_middle::mir::interpret::InterpError;
 use rustc_middle::ty;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_span::symbol::{sym, Symbol};
-use rustc_target::abi::{Abi, Scalar as ScalarAbi, Size, VariantIdx, Variants, WrappingRange};
+use rustc_target::abi::{
+    Abi, FieldIdx, Scalar as ScalarAbi, Size, VariantIdx, Variants, WrappingRange,
+};
 
 use std::hash::Hash;
 
@@ -269,14 +271,16 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                 match layout.variants {
                     Variants::Single { index } => {
                         // Inside a variant
-                        PathElem::Field(def.variant(index).fields[field].name)
+                        PathElem::Field(def.variant(index).fields[FieldIdx::from_usize(field)].name)
                     }
                     Variants::Multiple { .. } => bug!("we handled variants above"),
                 }
             }
 
             // other ADTs
-            ty::Adt(def, _) => PathElem::Field(def.non_enum_variant().fields[field].name),
+            ty::Adt(def, _) => {
+                PathElem::Field(def.non_enum_variant().fields[FieldIdx::from_usize(field)].name)
+            }
 
             // arrays/slices
             ty::Array(..) | ty::Slice(..) => PathElem::ArrayElem(field),
diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs
index ed9efe568..5ab389d04 100644
--- a/compiler/rustc_const_eval/src/lib.rs
+++ b/compiler/rustc_const_eval/src/lib.rs
@@ -20,7 +20,6 @@ Rust MIR: a lowered representation of Rust.
 #![feature(try_blocks)]
 #![feature(yeet_expr)]
 #![feature(if_let_guard)]
-#![feature(is_some_and)]
 #![recursion_limit = "256"]
 
 #[macro_use]
@@ -39,7 +38,7 @@ use rustc_macros::fluent_messages;
 use rustc_middle::ty;
 use rustc_middle::ty::query::Providers;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     const_eval::provide(providers);
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
index aa24d9053..55080d94f 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
@@ -246,7 +246,7 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> {
             self.check_local_or_return_ty(return_ty.skip_binder(), RETURN_PLACE);
         }
 
-        if !tcx.has_attr(def_id.to_def_id(), sym::rustc_do_not_const_check) {
+        if !tcx.has_attr(def_id, sym::rustc_do_not_const_check) {
             self.visit_body(&body);
         }
 
@@ -412,9 +412,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                         BorrowKind::Shallow => {
                             PlaceContext::NonMutatingUse(NonMutatingUseContext::ShallowBorrow)
                         }
-                        BorrowKind::Unique => {
-                            PlaceContext::NonMutatingUse(NonMutatingUseContext::UniqueBorrow)
-                        }
+                        BorrowKind::Unique => PlaceContext::MutatingUse(MutatingUseContext::Borrow),
                         BorrowKind::Mut { .. } => {
                             PlaceContext::MutatingUse(MutatingUseContext::Borrow)
                         }
@@ -553,7 +551,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             }
 
             Rvalue::Cast(CastKind::DynStar, _, _) => {
-                unimplemented!()
+                // `dyn*` coercion is implemented for CTFE.
             }
 
             Rvalue::Cast(_, _, _) => {}
@@ -643,7 +641,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                 if base_ty.is_unsafe_ptr() {
                     if proj_base.is_empty() {
                         let decl = &self.body.local_decls[place_local];
-                        if let Some(box LocalInfo::StaticRef { def_id, .. }) = decl.local_info {
+                        if let LocalInfo::StaticRef { def_id, .. } = *decl.local_info() {
                             let span = decl.source_info.span;
                             self.check_static(def_id, span);
                             return;
@@ -690,6 +688,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             | StatementKind::StorageLive(_)
             | StatementKind::StorageDead(_)
             | StatementKind::Retag { .. }
+            | StatementKind::PlaceMention(..)
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
@@ -721,6 +720,32 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     }
                 };
 
+                // Check that all trait bounds that are marked as `~const` can be satisfied.
+                //
+                // Typeck only does a "non-const" check since it operates on HIR and cannot distinguish
+                // which path expressions are getting called on and which path expressions are only used
+                // as function pointers. This is required for correctness.
+                let infcx = tcx.infer_ctxt().build();
+                let ocx = ObligationCtxt::new(&infcx);
+
+                let predicates = tcx.predicates_of(callee).instantiate(tcx, substs);
+                let cause = ObligationCause::new(
+                    terminator.source_info.span,
+                    self.body.source.def_id().expect_local(),
+                    ObligationCauseCode::ItemObligation(callee),
+                );
+                let normalized_predicates = ocx.normalize(&cause, param_env, predicates);
+                ocx.register_obligations(traits::predicates_for_generics(
+                    |_, _| cause.clone(),
+                    self.param_env,
+                    normalized_predicates,
+                ));
+
+                let errors = ocx.select_all_or_error();
+                if !errors.is_empty() {
+                    infcx.err_ctxt().report_fulfillment_errors(&errors);
+                }
+
                 // Attempting to call a trait method?
                 if let Some(trait_id) = tcx.trait_of_item(callee) {
                     trace!("attempting to call a trait method");
@@ -748,31 +773,6 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                         selcx.select(&obligation)
                     };
 
-                    // do a well-formedness check on the trait method being called. This is because typeck only does a
-                    // "non-const" check. This is required for correctness here.
-                    {
-                        let infcx = tcx.infer_ctxt().build();
-                        let ocx = ObligationCtxt::new(&infcx);
-
-                        let predicates = tcx.predicates_of(callee).instantiate(tcx, substs);
-                        let cause = ObligationCause::new(
-                            terminator.source_info.span,
-                            self.body.source.def_id().expect_local(),
-                            ObligationCauseCode::ItemObligation(callee),
-                        );
-                        let normalized_predicates = ocx.normalize(&cause, param_env, predicates);
-                        ocx.register_obligations(traits::predicates_for_generics(
-                            |_, _| cause.clone(),
-                            self.param_env,
-                            normalized_predicates,
-                        ));
-
-                        let errors = ocx.select_all_or_error();
-                        if !errors.is_empty() {
-                            infcx.err_ctxt().report_fulfillment_errors(&errors, None);
-                        }
-                    }
-
                     match implsrc {
                         Ok(Some(ImplSource::Param(_, ty::BoundConstness::ConstIfConst))) => {
                             debug!(
@@ -926,15 +926,24 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
                 // If the `const fn` we are trying to call is not const-stable, ensure that we have
                 // the proper feature gate enabled.
-                if let Some(gate) = is_unstable_const_fn(tcx, callee) {
+                if let Some((gate, implied_by)) = is_unstable_const_fn(tcx, callee) {
                     trace!(?gate, "calling unstable const fn");
                     if self.span.allows_unstable(gate) {
                         return;
                     }
+                    if let Some(implied_by_gate) = implied_by && self.span.allows_unstable(implied_by_gate) {
+                        return;
+                    }
 
                     // Calling an unstable function *always* requires that the corresponding gate
-                    // be enabled, even if the function has `#[rustc_allow_const_fn_unstable(the_gate)]`.
-                    if !tcx.features().declared_lib_features.iter().any(|&(sym, _)| sym == gate) {
+                    // (or implied gate) be enabled, even if the function has
+                    // `#[rustc_allow_const_fn_unstable(the_gate)]`.
+                    let gate_declared = |gate| {
+                        tcx.features().declared_lib_features.iter().any(|&(sym, _)| sym == gate)
+                    };
+                    let feature_gate_declared = gate_declared(gate);
+                    let implied_gate_declared = implied_by.map(gate_declared).unwrap_or(false);
+                    if !feature_gate_declared && !implied_gate_declared {
                         self.check_op(ops::FnCallUnstable(callee, Some(gate)));
                         return;
                     }
@@ -947,7 +956,6 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     }
 
                     // Otherwise, we are something const-stable calling a const-unstable fn.
-
                     if super::rustc_allow_const_fn_unstable(tcx, caller, gate) {
                         trace!("rustc_allow_const_fn_unstable gate active");
                         return;
@@ -977,8 +985,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
             // Forbid all `Drop` terminators unless the place being dropped is a local with no
             // projections that cannot be `NeedsNonConstDrop`.
-            TerminatorKind::Drop { place: dropped_place, .. }
-            | TerminatorKind::DropAndReplace { place: dropped_place, .. } => {
+            TerminatorKind::Drop { place: dropped_place, .. } => {
                 // If we are checking live drops after drop-elaboration, don't emit duplicate
                 // errors here.
                 if super::post_drop_elaboration::checking_enabled(self.ccx) {
@@ -1022,9 +1029,9 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                 self.check_op(ops::Generator(hir::GeneratorKind::Gen))
             }
 
-            TerminatorKind::Abort => {
+            TerminatorKind::Terminate => {
                 // Cleanup blocks are skipped for const checking (see `visit_basic_block_data`).
-                span_bug!(self.span, "`Abort` terminator outside of cleanup block")
+                span_bug!(self.span, "`Terminate` terminator outside of cleanup block")
             }
 
             TerminatorKind::Assert { .. }
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
index 3e416b89c..c0f5b3725 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
@@ -12,9 +12,7 @@ use rustc_infer::traits::{ImplSource, Obligation, ObligationCause};
 use rustc_middle::mir;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
-use rustc_middle::ty::{
-    suggest_constraining_type_param, Adt, Closure, DefIdTree, FnDef, FnPtr, Param, Ty,
-};
+use rustc_middle::ty::{suggest_constraining_type_param, Adt, Closure, FnDef, FnPtr, Param, Ty};
 use rustc_middle::ty::{Binder, TraitRef};
 use rustc_session::parse::feature_err;
 use rustc_span::symbol::sym;
@@ -706,7 +704,7 @@ pub mod ty {
 
         fn importance(&self) -> DiagnosticImportance {
             match self.0 {
-                mir::LocalKind::Var | mir::LocalKind::Temp => DiagnosticImportance::Secondary,
+                mir::LocalKind::Temp => DiagnosticImportance::Secondary,
                 mir::LocalKind::ReturnPointer | mir::LocalKind::Arg => {
                     DiagnosticImportance::Primary
                 }
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
index cf4e875c9..1f1640fd8 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
@@ -30,7 +30,7 @@ pub fn check_live_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mir::Body<'tcx>) {
         return;
     }
 
-    if tcx.has_attr(def_id.to_def_id(), sym::rustc_do_not_const_check) {
+    if tcx.has_attr(def_id, sym::rustc_do_not_const_check) {
         return;
     }
 
@@ -80,8 +80,7 @@ impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> {
         trace!("visit_terminator: terminator={:?} location={:?}", terminator, location);
 
         match &terminator.kind {
-            mir::TerminatorKind::Drop { place: dropped_place, .. }
-            | mir::TerminatorKind::DropAndReplace { place: dropped_place, .. } => {
+            mir::TerminatorKind::Drop { place: dropped_place, .. } => {
                 let dropped_ty = dropped_place.ty(self.body, self.tcx).ty;
                 if !NeedsNonConstDrop::in_any_value_of_ty(self.ccx, dropped_ty) {
                     // Instead of throwing a bug, we just return here. This is because we have to
@@ -105,7 +104,7 @@ impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> {
                 }
             }
 
-            mir::TerminatorKind::Abort
+            mir::TerminatorKind::Terminate
             | mir::TerminatorKind::Call { .. }
             | mir::TerminatorKind::Assert { .. }
             | mir::TerminatorKind::FalseEdge { .. }
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
index bb4b7ad50..6758cba2e 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
@@ -9,7 +9,7 @@ use rustc_middle::mir;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, subst::SubstsRef, AdtDef, Ty};
 use rustc_trait_selection::traits::{
-    self, ImplSource, Obligation, ObligationCause, SelectionContext,
+    self, ImplSource, Obligation, ObligationCause, ObligationCtxt, SelectionContext,
 };
 
 use super::ConstCx;
@@ -184,7 +184,10 @@ impl Qualif for NeedsNonConstDrop {
         }
 
         // If we had any errors, then it's bad
-        !traits::fully_solve_obligations(&infcx, impl_src.nested_obligations()).is_empty()
+        let ocx = ObligationCtxt::new(&infcx);
+        ocx.register_obligations(impl_src.nested_obligations());
+        let errors = ocx.select_all_or_error();
+        !errors.is_empty()
     }
 
     fn in_adt_inherently<'tcx>(
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs b/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs
index 805e6096b..78c74e189 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs
@@ -222,23 +222,8 @@ where
         // The effect of assignment to the return place in `TerminatorKind::Call` is not applied
         // here; that occurs in `apply_call_return_effect`.
 
-        if let mir::TerminatorKind::DropAndReplace { value, place, .. } = &terminator.kind {
-            let qualif = qualifs::in_operand::<Q, _>(
-                self.ccx,
-                &mut |l| self.state.qualif.contains(l),
-                value,
-            );
-
-            if !place.is_indirect() {
-                self.assign_qualif_direct(place, qualif);
-            }
-        }
-
         // We ignore borrow on drop because custom drop impls are not allowed in consts.
         // FIXME: Reconsider if accounting for borrows in drops is necessary for const drop.
-
-        // We need to assign qualifs to the dropped location before visiting the operand that
-        // replaces it since qualifs can be cleared on move.
         self.super_terminator(terminator, location);
     }
 }
diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs
index 3f3b66b06..7919aed09 100644
--- a/compiler/rustc_const_eval/src/transform/promote_consts.rs
+++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs
@@ -21,7 +21,7 @@ use rustc_middle::ty::subst::InternalSubsts;
 use rustc_middle::ty::{self, List, TyCtxt, TypeVisitableExt};
 use rustc_span::Span;
 
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{Idx, IndexSlice, IndexVec};
 
 use std::cell::Cell;
 use std::{cmp, iter, mem};
@@ -106,8 +106,9 @@ impl<'tcx> Visitor<'tcx> for Collector<'_, 'tcx> {
         debug!("visit_local: index={:?} context={:?} location={:?}", index, context, location);
         // We're only interested in temporaries and the return place
         match self.ccx.body.local_kind(index) {
-            LocalKind::Temp | LocalKind::ReturnPointer => {}
-            LocalKind::Arg | LocalKind::Var => return,
+            LocalKind::Arg => return,
+            LocalKind::Temp if self.ccx.body.local_decls[index].is_user_variable() => return,
+            LocalKind::ReturnPointer | LocalKind::Temp => {}
         }
 
         // Ignore drops, if the temp gets promoted,
@@ -183,7 +184,7 @@ pub fn collect_temps_and_candidates<'tcx>(
 /// This wraps an `Item`, and has access to all fields of that `Item` via `Deref` coercion.
 struct Validator<'a, 'tcx> {
     ccx: &'a ConstCx<'a, 'tcx>,
-    temps: &'a mut IndexVec<Local, TempState>,
+    temps: &'a mut IndexSlice<Local, TempState>,
 }
 
 impl<'a, 'tcx> std::ops::Deref for Validator<'a, 'tcx> {
@@ -668,7 +669,7 @@ impl<'tcx> Validator<'_, 'tcx> {
 // FIXME(eddyb) remove the differences for promotability in `static`, `const`, `const fn`.
 pub fn validate_candidates(
     ccx: &ConstCx<'_, '_>,
-    temps: &mut IndexVec<Local, TempState>,
+    temps: &mut IndexSlice<Local, TempState>,
     candidates: &[Candidate],
 ) -> Vec<Candidate> {
     let mut validator = Validator { ccx, temps };
@@ -706,7 +707,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
     }
 
     fn assign(&mut self, dest: Local, rvalue: Rvalue<'tcx>, span: Span) {
-        let last = self.promoted.basic_blocks.last().unwrap();
+        let last = self.promoted.basic_blocks.last_index().unwrap();
         let data = &mut self.promoted[last];
         data.statements.push(Statement {
             source_info: SourceInfo::outermost(span),
@@ -799,14 +800,14 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                         self.visit_operand(arg, loc);
                     }
 
-                    let last = self.promoted.basic_blocks.last().unwrap();
+                    let last = self.promoted.basic_blocks.last_index().unwrap();
                     let new_target = self.new_block();
 
                     *self.promoted[last].terminator_mut() = Terminator {
                         kind: TerminatorKind::Call {
                             func,
                             args,
-                            cleanup: None,
+                            unwind: UnwindAction::Continue,
                             destination: Place::from(new_temp),
                             target: Some(new_target),
                             from_hir_call,
diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
index fb37eb79a..d4bed9738 100644
--- a/compiler/rustc_const_eval/src/transform/validate.rs
+++ b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -5,19 +5,18 @@ use rustc_index::bit_set::BitSet;
 use rustc_index::vec::IndexVec;
 use rustc_infer::traits::Reveal;
 use rustc_middle::mir::interpret::Scalar;
-use rustc_middle::mir::visit::NonUseContext::VarDebugInfo;
-use rustc_middle::mir::visit::{PlaceContext, Visitor};
+use rustc_middle::mir::visit::{NonUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::{
     traversal, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping, Local, Location,
     MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef, ProjectionElem,
     RetagKind, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, Terminator,
-    TerminatorKind, UnOp, START_BLOCK,
+    TerminatorKind, UnOp, UnwindAction, VarDebugInfo, VarDebugInfoContents, START_BLOCK,
 };
 use rustc_middle::ty::{self, InstanceDef, ParamEnv, Ty, TyCtxt, TypeVisitableExt};
 use rustc_mir_dataflow::impls::MaybeStorageLive;
 use rustc_mir_dataflow::storage::always_storage_live_locals;
 use rustc_mir_dataflow::{Analysis, ResultsCursor};
-use rustc_target::abi::{Size, VariantIdx};
+use rustc_target::abi::{Size, FIRST_VARIANT};
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 enum EdgeKind {
@@ -72,6 +71,17 @@ impl<'tcx> MirPass<'tcx> for Validator {
         };
         checker.visit_body(body);
         checker.check_cleanup_control_flow();
+
+        if let MirPhase::Runtime(_) = body.phase {
+            if let ty::InstanceDef::Item(_) = body.source.instance {
+                if body.has_free_regions() {
+                    checker.fail(
+                        Location::START,
+                        format!("Free regions in optimized {} MIR", body.phase.name()),
+                    );
+                }
+            }
+        }
     }
 }
 
@@ -222,6 +232,24 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         }
     }
 
+    fn check_unwind_edge(&mut self, location: Location, unwind: UnwindAction) {
+        let is_cleanup = self.body.basic_blocks[location.block].is_cleanup;
+        match unwind {
+            UnwindAction::Cleanup(unwind) => {
+                if is_cleanup {
+                    self.fail(location, "unwind on cleanup block");
+                }
+                self.check_edge(location, unwind, EdgeKind::Unwind);
+            }
+            UnwindAction::Continue => {
+                if is_cleanup {
+                    self.fail(location, "unwind on cleanup block");
+                }
+            }
+            UnwindAction::Unreachable | UnwindAction::Terminate => (),
+        }
+    }
+
     /// Check if src can be assigned into dest.
     /// This is not precise, it will accept some incorrect assignments.
     fn mir_assign_valid_types(&self, src: Ty<'tcx>, dest: Ty<'tcx>) -> bool {
@@ -348,8 +376,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                         check_equal(self, location, *f_ty);
                     }
                     ty::Adt(adt_def, substs) => {
-                        let var = parent_ty.variant_index.unwrap_or(VariantIdx::from_u32(0));
-                        let Some(field) = adt_def.variant(var).fields.get(f.as_usize()) else {
+                        let var = parent_ty.variant_index.unwrap_or(FIRST_VARIANT);
+                        let Some(field) = adt_def.variant(var).fields.get(f) else {
                             fail_out_of_bounds(self, location);
                             return;
                         };
@@ -408,13 +436,49 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
         self.super_projection_elem(local, proj_base, elem, context, location);
     }
 
+    fn visit_var_debug_info(&mut self, debuginfo: &VarDebugInfo<'tcx>) {
+        let check_place = |place: Place<'_>| {
+            if place.projection.iter().any(|p| !p.can_use_in_debuginfo()) {
+                self.fail(
+                    START_BLOCK.start_location(),
+                    format!("illegal place {:?} in debuginfo for {:?}", place, debuginfo.name),
+                );
+            }
+        };
+        match debuginfo.value {
+            VarDebugInfoContents::Const(_) => {}
+            VarDebugInfoContents::Place(place) => check_place(place),
+            VarDebugInfoContents::Composite { ty, ref fragments } => {
+                for f in fragments {
+                    check_place(f.contents);
+                    if ty.is_union() || ty.is_enum() {
+                        self.fail(
+                            START_BLOCK.start_location(),
+                            format!("invalid type {:?} for composite debuginfo", ty),
+                        );
+                    }
+                    if f.projection.iter().any(|p| !matches!(p, PlaceElem::Field(..))) {
+                        self.fail(
+                            START_BLOCK.start_location(),
+                            format!(
+                                "illegal projection {:?} in debuginfo for {:?}",
+                                f.projection, debuginfo.name
+                            ),
+                        );
+                    }
+                }
+            }
+        }
+        self.super_var_debug_info(debuginfo);
+    }
+
     fn visit_place(&mut self, place: &Place<'tcx>, cntxt: PlaceContext, location: Location) {
         // Set off any `bug!`s in the type computation code
         let _ = place.ty(&self.body.local_decls, self.tcx);
 
         if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial)
             && place.projection.len() > 1
-            && cntxt != PlaceContext::NonUse(VarDebugInfo)
+            && cntxt != PlaceContext::NonUse(NonUseContext::VarDebugInfo)
             && place.projection[1..].contains(&ProjectionElem::Deref)
         {
             self.fail(location, format!("{:?}, has deref at the wrong place", place));
@@ -553,15 +617,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                             );
                         }
                     }
-                    Shl | Shr => {
-                        for x in [a, b] {
-                            check_kinds!(
-                                x,
-                                "Cannot perform checked shift on non-integer type {:?}",
-                                ty::Uint(..) | ty::Int(..)
-                            )
-                        }
-                    }
                     _ => self.fail(location, format!("There is no checked version of {:?}", op)),
                 }
             }
@@ -619,6 +674,41 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                             );
                         }
                     }
+                    CastKind::Transmute => {
+                        if let MirPhase::Runtime(..) = self.mir_phase {
+                            // Unlike `mem::transmute`, a MIR `Transmute` is well-formed
+                            // for any two `Sized` types, just potentially UB to run.
+
+                            if !self
+                                .tcx
+                                .normalize_erasing_regions(self.param_env, op_ty)
+                                .is_sized(self.tcx, self.param_env)
+                            {
+                                self.fail(
+                                    location,
+                                    format!("Cannot transmute from non-`Sized` type {op_ty:?}"),
+                                );
+                            }
+                            if !self
+                                .tcx
+                                .normalize_erasing_regions(self.param_env, *target_type)
+                                .is_sized(self.tcx, self.param_env)
+                            {
+                                self.fail(
+                                    location,
+                                    format!("Cannot transmute to non-`Sized` type {target_type:?}"),
+                                );
+                            }
+                        } else {
+                            self.fail(
+                                location,
+                                format!(
+                                    "Transmute is not supported in non-runtime phase {:?}.",
+                                    self.mir_phase
+                                ),
+                            );
+                        }
+                    }
                 }
             }
             Rvalue::Repeat(_, _)
@@ -648,8 +738,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     );
                 }
                 if let Rvalue::CopyForDeref(place) = rvalue {
-                    if !place.ty(&self.body.local_decls, self.tcx).ty.builtin_deref(true).is_some()
-                    {
+                    if place.ty(&self.body.local_decls, self.tcx).ty.builtin_deref(true).is_none() {
                         self.fail(
                             location,
                             "`CopyForDeref` should only be used for dereferenceable types",
@@ -668,6 +757,14 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     }
                 }
             }
+            StatementKind::PlaceMention(..) => {
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
+                    self.fail(
+                        location,
+                        "`PlaceMention` should have been removed after drop lowering phase",
+                    );
+                }
+            }
             StatementKind::AscribeUserType(..) => {
                 if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(
@@ -831,23 +928,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             }
             TerminatorKind::Drop { target, unwind, .. } => {
                 self.check_edge(location, *target, EdgeKind::Normal);
-                if let Some(unwind) = unwind {
-                    self.check_edge(location, *unwind, EdgeKind::Unwind);
-                }
-            }
-            TerminatorKind::DropAndReplace { target, unwind, .. } => {
-                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
-                    self.fail(
-                        location,
-                        "`DropAndReplace` should have been removed during drop elaboration",
-                    );
-                }
-                self.check_edge(location, *target, EdgeKind::Normal);
-                if let Some(unwind) = unwind {
-                    self.check_edge(location, *unwind, EdgeKind::Unwind);
-                }
+                self.check_unwind_edge(location, *unwind);
             }
-            TerminatorKind::Call { func, args, destination, target, cleanup, .. } => {
+            TerminatorKind::Call { func, args, destination, target, unwind, .. } => {
                 let func_ty = func.ty(&self.body.local_decls, self.tcx);
                 match func_ty.kind() {
                     ty::FnPtr(..) | ty::FnDef(..) => {}
@@ -859,9 +942,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 if let Some(target) = target {
                     self.check_edge(location, *target, EdgeKind::Normal);
                 }
-                if let Some(cleanup) = cleanup {
-                    self.check_edge(location, *cleanup, EdgeKind::Unwind);
-                }
+                self.check_unwind_edge(location, *unwind);
 
                 // The call destination place and Operand::Move place used as an argument might be
                 // passed by a reference to the callee. Consequently they must be non-overlapping.
@@ -887,7 +968,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     );
                 }
             }
-            TerminatorKind::Assert { cond, target, cleanup, .. } => {
+            TerminatorKind::Assert { cond, target, unwind, .. } => {
                 let cond_ty = cond.ty(&self.body.local_decls, self.tcx);
                 if cond_ty != self.tcx.types.bool {
                     self.fail(
@@ -899,9 +980,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     );
                 }
                 self.check_edge(location, *target, EdgeKind::Normal);
-                if let Some(cleanup) = cleanup {
-                    self.check_edge(location, *cleanup, EdgeKind::Unwind);
-                }
+                self.check_unwind_edge(location, *unwind);
             }
             TerminatorKind::Yield { resume, drop, .. } => {
                 if self.body.generator.is_none() {
@@ -933,17 +1012,13 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     );
                 }
                 self.check_edge(location, *real_target, EdgeKind::Normal);
-                if let Some(unwind) = unwind {
-                    self.check_edge(location, *unwind, EdgeKind::Unwind);
-                }
+                self.check_unwind_edge(location, *unwind);
             }
-            TerminatorKind::InlineAsm { destination, cleanup, .. } => {
+            TerminatorKind::InlineAsm { destination, unwind, .. } => {
                 if let Some(destination) = destination {
                     self.check_edge(location, *destination, EdgeKind::Normal);
                 }
-                if let Some(cleanup) = cleanup {
-                    self.check_edge(location, *cleanup, EdgeKind::Unwind);
-                }
+                self.check_unwind_edge(location, *unwind);
             }
             TerminatorKind::GeneratorDrop => {
                 if self.body.generator.is_none() {
@@ -956,10 +1031,13 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     );
                 }
             }
-            TerminatorKind::Resume | TerminatorKind::Abort => {
+            TerminatorKind::Resume | TerminatorKind::Terminate => {
                 let bb = location.block;
                 if !self.body.basic_blocks[bb].is_cleanup {
-                    self.fail(location, "Cannot `Resume` or `Abort` from non-cleanup basic block")
+                    self.fail(
+                        location,
+                        "Cannot `Resume` or `Terminate` from non-cleanup basic block",
+                    )
                 }
             }
             TerminatorKind::Return => {
diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
index 29cb2c0a3..2102f09c5 100644
--- a/compiler/rustc_data_structures/Cargo.toml
+++ b/compiler/rustc_data_structures/Cargo.toml
@@ -9,18 +9,19 @@ edition = "2021"
 arrayvec = { version = "0.7", default-features = false }
 bitflags = "1.2.1"
 cfg-if = "1.0"
-ena = "0.14.1"
-indexmap = { version = "1.9.1" }
+ena = "0.14.2"
+indexmap = { version = "1.9.3" }
 jobserver_crate = { version = "0.1.13", package = "jobserver" }
 libc = "0.2"
 measureme = "10.0.0"
-rayon-core = { version = "0.4.0", package = "rustc-rayon-core", optional = true }
-rayon = { version = "0.4.0", package = "rustc-rayon", optional = true }
+rustc-rayon-core = { version = "0.5.0", optional = true }
+rustc-rayon = { version = "0.5.0", optional = true }
 rustc_graphviz = { path = "../rustc_graphviz" }
 rustc-hash = "1.1.0"
 rustc_index = { path = "../rustc_index", package = "rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
+serde_json = "1.0.59"
 smallvec = { version = "1.8.1", features = [
     "const_generics",
     "union",
@@ -31,16 +32,24 @@ stacker = "0.1.15"
 tempfile = "3.2"
 thin-vec = "0.2.12"
 tracing = "0.1"
-elsa = "1.8"
+elsa = "=1.7.1"
+itertools = "0.10.1"
 
 [dependencies.parking_lot]
 version = "0.11"
 
-[target.'cfg(windows)'.dependencies]
-winapi = { version = "0.3", features = ["fileapi", "psapi", "winerror"] }
+[target.'cfg(windows)'.dependencies.windows]
+version = "0.46.0"
+features = [
+    "Win32_Foundation",
+    "Win32_Storage_FileSystem",
+    "Win32_System_IO",
+    "Win32_System_ProcessStatus",
+    "Win32_System_Threading",
+]
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 memmap2 = "0.2.1"
 
 [features]
-rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "rayon", "rayon-core"]
+rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "rustc-rayon", "rustc-rayon-core"]
diff --git a/compiler/rustc_data_structures/src/map_in_place.rs b/compiler/rustc_data_structures/src/flat_map_in_place.rs
index a0d4b7ade..f58844f28 100644
--- a/compiler/rustc_data_structures/src/map_in_place.rs
+++ b/compiler/rustc_data_structures/src/flat_map_in_place.rs
@@ -2,14 +2,7 @@ use smallvec::{Array, SmallVec};
 use std::ptr;
 use thin_vec::ThinVec;
 
-pub trait MapInPlace<T>: Sized {
-    fn map_in_place<F>(&mut self, mut f: F)
-    where
-        F: FnMut(T) -> T,
-    {
-        self.flat_map_in_place(|e| Some(f(e)))
-    }
-
+pub trait FlatMapInPlace<T>: Sized {
     fn flat_map_in_place<F, I>(&mut self, f: F)
     where
         F: FnMut(T) -> I,
@@ -66,14 +59,14 @@ macro_rules! flat_map_in_place {
     };
 }
 
-impl<T> MapInPlace<T> for Vec<T> {
+impl<T> FlatMapInPlace<T> for Vec<T> {
     flat_map_in_place!();
 }
 
-impl<T, A: Array<Item = T>> MapInPlace<T> for SmallVec<A> {
+impl<T, A: Array<Item = T>> FlatMapInPlace<T> for SmallVec<A> {
     flat_map_in_place!();
 }
 
-impl<T> MapInPlace<T> for ThinVec<T> {
+impl<T> FlatMapInPlace<T> for ThinVec<T> {
     flat_map_in_place!();
 }
diff --git a/compiler/rustc_data_structures/src/flock.rs b/compiler/rustc_data_structures/src/flock.rs
index e395d8dbb..efdb44248 100644
--- a/compiler/rustc_data_structures/src/flock.rs
+++ b/compiler/rustc_data_structures/src/flock.rs
@@ -4,9 +4,6 @@
 //! green/native threading. This is just a bare-bones enough solution for
 //! librustdoc, it is not production quality at all.
 
-#![allow(non_camel_case_types)]
-#![allow(nonstandard_style)]
-
 cfg_if! {
     if #[cfg(target_os = "linux")] {
         mod linux;
@@ -16,7 +13,7 @@ cfg_if! {
         use unix as imp;
     } else if #[cfg(windows)] {
         mod windows;
-        use windows as imp;
+        use self::windows as imp;
     } else {
         mod unsupported;
         use unsupported as imp;
diff --git a/compiler/rustc_data_structures/src/flock/windows.rs b/compiler/rustc_data_structures/src/flock/windows.rs
index 43e6caaa1..da128f464 100644
--- a/compiler/rustc_data_structures/src/flock/windows.rs
+++ b/compiler/rustc_data_structures/src/flock/windows.rs
@@ -1,13 +1,16 @@
 use std::fs::{File, OpenOptions};
 use std::io;
-use std::mem;
 use std::os::windows::prelude::*;
 use std::path::Path;
 
-use winapi::shared::winerror::ERROR_INVALID_FUNCTION;
-use winapi::um::fileapi::LockFileEx;
-use winapi::um::minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY, OVERLAPPED};
-use winapi::um::winnt::{FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE};
+use windows::{
+    Win32::Foundation::{ERROR_INVALID_FUNCTION, HANDLE},
+    Win32::Storage::FileSystem::{
+        LockFileEx, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, LOCKFILE_EXCLUSIVE_LOCK,
+        LOCKFILE_FAIL_IMMEDIATELY, LOCK_FILE_FLAGS,
+    },
+    Win32::System::IO::OVERLAPPED,
+};
 
 #[derive(Debug)]
 pub struct Lock {
@@ -25,7 +28,7 @@ impl Lock {
         let share_mode = FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE;
 
         let mut open_options = OpenOptions::new();
-        open_options.read(true).share_mode(share_mode);
+        open_options.read(true).share_mode(share_mode.0);
 
         if create {
             open_options.create(true).write(true);
@@ -43,33 +46,42 @@ impl Lock {
             }
         };
 
-        let ret = unsafe {
-            let mut overlapped: OVERLAPPED = mem::zeroed();
+        let mut flags = LOCK_FILE_FLAGS::default();
+        if !wait {
+            flags |= LOCKFILE_FAIL_IMMEDIATELY;
+        }
 
-            let mut dwFlags = 0;
-            if !wait {
-                dwFlags |= LOCKFILE_FAIL_IMMEDIATELY;
-            }
+        if exclusive {
+            flags |= LOCKFILE_EXCLUSIVE_LOCK;
+        }
 
-            if exclusive {
-                dwFlags |= LOCKFILE_EXCLUSIVE_LOCK;
-            }
+        let mut overlapped = OVERLAPPED::default();
 
-            debug!("attempting to acquire lock on lock file `{}`", p.display());
-            LockFileEx(file.as_raw_handle(), dwFlags, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, &mut overlapped)
-        };
-        if ret == 0 {
-            let err = io::Error::last_os_error();
-            debug!("failed acquiring file lock: {}", err);
-            Err(err)
-        } else {
-            debug!("successfully acquired lock");
-            Ok(Lock { _file: file })
+        debug!("attempting to acquire lock on lock file `{}`", p.display());
+
+        unsafe {
+            LockFileEx(
+                HANDLE(file.as_raw_handle() as isize),
+                flags,
+                0,
+                u32::MAX,
+                u32::MAX,
+                &mut overlapped,
+            )
         }
+        .ok()
+        .map_err(|e| {
+            let err = io::Error::from_raw_os_error(e.code().0);
+            debug!("failed acquiring file lock: {}", err);
+            err
+        })?;
+
+        debug!("successfully acquired lock");
+        Ok(Lock { _file: file })
     }
 
     pub fn error_unsupported(err: &io::Error) -> bool {
-        err.raw_os_error() == Some(ERROR_INVALID_FUNCTION as i32)
+        err.raw_os_error() == Some(ERROR_INVALID_FUNCTION.0 as i32)
     }
 }
 
diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
index 0a21a4249..0df9dc112 100644
--- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
@@ -10,7 +10,7 @@
 //! <https://www.cs.princeton.edu/courses/archive/spr03/cs423/download/dominators.pdf>
 
 use super::ControlFlowGraph;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{Idx, IndexSlice, IndexVec};
 use std::cmp::Ordering;
 
 #[cfg(test)]
@@ -256,10 +256,10 @@ pub fn dominators<G: ControlFlowGraph>(graph: G) -> Dominators<G::Node> {
 /// where `+>` is a proper ancestor and `*>` is just an ancestor.
 #[inline]
 fn eval(
-    ancestor: &mut IndexVec<PreorderIndex, PreorderIndex>,
+    ancestor: &mut IndexSlice<PreorderIndex, PreorderIndex>,
     lastlinked: Option<PreorderIndex>,
-    semi: &IndexVec<PreorderIndex, PreorderIndex>,
-    label: &mut IndexVec<PreorderIndex, PreorderIndex>,
+    semi: &IndexSlice<PreorderIndex, PreorderIndex>,
+    label: &mut IndexSlice<PreorderIndex, PreorderIndex>,
     node: PreorderIndex,
 ) -> PreorderIndex {
     if is_processed(node, lastlinked) {
@@ -277,10 +277,10 @@ fn is_processed(v: PreorderIndex, lastlinked: Option<PreorderIndex>) -> bool {
 
 #[inline]
 fn compress(
-    ancestor: &mut IndexVec<PreorderIndex, PreorderIndex>,
+    ancestor: &mut IndexSlice<PreorderIndex, PreorderIndex>,
     lastlinked: Option<PreorderIndex>,
-    semi: &IndexVec<PreorderIndex, PreorderIndex>,
-    label: &mut IndexVec<PreorderIndex, PreorderIndex>,
+    semi: &IndexSlice<PreorderIndex, PreorderIndex>,
+    label: &mut IndexSlice<PreorderIndex, PreorderIndex>,
     v: PreorderIndex,
 ) {
     assert!(is_processed(v, lastlinked));
diff --git a/compiler/rustc_data_structures/src/graph/implementation/mod.rs b/compiler/rustc_data_structures/src/graph/implementation/mod.rs
index 1aa7ac024..9ff401c3c 100644
--- a/compiler/rustc_data_structures/src/graph/implementation/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/implementation/mod.rs
@@ -206,17 +206,11 @@ impl<N: Debug, E: Debug> Graph<N, E> {
         AdjacentEdges { graph: self, direction, next: first_edge }
     }
 
-    pub fn successor_nodes<'a>(
-        &'a self,
-        source: NodeIndex,
-    ) -> impl Iterator<Item = NodeIndex> + 'a {
+    pub fn successor_nodes(&self, source: NodeIndex) -> impl Iterator<Item = NodeIndex> + '_ {
         self.outgoing_edges(source).targets()
     }
 
-    pub fn predecessor_nodes<'a>(
-        &'a self,
-        target: NodeIndex,
-    ) -> impl Iterator<Item = NodeIndex> + 'a {
+    pub fn predecessor_nodes(&self, target: NodeIndex) -> impl Iterator<Item = NodeIndex> + '_ {
         self.incoming_edges(target).sources()
     }
 
diff --git a/compiler/rustc_data_structures/src/graph/iterate/mod.rs b/compiler/rustc_data_structures/src/graph/iterate/mod.rs
index 8a9af300c..01a83b40a 100644
--- a/compiler/rustc_data_structures/src/graph/iterate/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/iterate/mod.rs
@@ -1,6 +1,6 @@
 use super::{DirectedGraph, WithNumNodes, WithStartNode, WithSuccessors};
 use rustc_index::bit_set::BitSet;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use std::ops::ControlFlow;
 
 #[cfg(test)]
@@ -31,7 +31,7 @@ fn post_order_walk<G: DirectedGraph + WithSuccessors + WithNumNodes>(
     graph: &G,
     node: G::Node,
     result: &mut Vec<G::Node>,
-    visited: &mut IndexVec<G::Node, bool>,
+    visited: &mut IndexSlice<G::Node, bool>,
 ) {
     struct PostOrderFrame<Node, Iter> {
         node: Node,
diff --git a/compiler/rustc_data_structures/src/graph/scc/mod.rs b/compiler/rustc_data_structures/src/graph/scc/mod.rs
index c4b11951a..28c357e54 100644
--- a/compiler/rustc_data_structures/src/graph/scc/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/scc/mod.rs
@@ -8,7 +8,7 @@
 use crate::fx::FxHashSet;
 use crate::graph::vec_graph::VecGraph;
 use crate::graph::{DirectedGraph, GraphSuccessors, WithNumEdges, WithNumNodes, WithSuccessors};
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{Idx, IndexSlice, IndexVec};
 use std::ops::Range;
 
 #[cfg(test)]
@@ -43,7 +43,7 @@ impl<N: Idx, S: Idx + Ord> Sccs<N, S> {
         SccsConstruction::construct(graph)
     }
 
-    pub fn scc_indices(&self) -> &IndexVec<N, S> {
+    pub fn scc_indices(&self) -> &IndexSlice<N, S> {
         &self.scc_indices
     }
 
@@ -123,7 +123,7 @@ impl<S: Idx> SccData<S> {
         self.ranges.len()
     }
 
-    pub fn ranges(&self) -> &IndexVec<S, Range<usize>> {
+    pub fn ranges(&self) -> &IndexSlice<S, Range<usize>> {
         &self.ranges
     }
 
diff --git a/compiler/rustc_data_structures/src/graph/scc/tests.rs b/compiler/rustc_data_structures/src/graph/scc/tests.rs
index 820a70fc8..513df666d 100644
--- a/compiler/rustc_data_structures/src/graph/scc/tests.rs
+++ b/compiler/rustc_data_structures/src/graph/scc/tests.rs
@@ -56,7 +56,7 @@ fn test_three_sccs() {
     assert_eq!(sccs.scc(1), 0);
     assert_eq!(sccs.scc(2), 0);
     assert_eq!(sccs.scc(3), 2);
-    assert_eq!(sccs.successors(0), &[]);
+    assert_eq!(sccs.successors(0), &[] as &[usize]);
     assert_eq!(sccs.successors(1), &[0]);
     assert_eq!(sccs.successors(2), &[0]);
 }
@@ -113,7 +113,7 @@ fn test_find_state_2() {
     assert_eq!(sccs.scc(2), 0);
     assert_eq!(sccs.scc(3), 0);
     assert_eq!(sccs.scc(4), 0);
-    assert_eq!(sccs.successors(0), &[]);
+    assert_eq!(sccs.successors(0), &[] as &[usize]);
 }
 
 #[test]
@@ -138,7 +138,7 @@ fn test_find_state_3() {
     assert_eq!(sccs.scc(3), 0);
     assert_eq!(sccs.scc(4), 0);
     assert_eq!(sccs.scc(5), 1);
-    assert_eq!(sccs.successors(0), &[]);
+    assert_eq!(sccs.successors(0), &[] as &[usize]);
     assert_eq!(sccs.successors(1), &[0]);
 }
 
diff --git a/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs b/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs
index c8f979267..7c866da60 100644
--- a/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs
+++ b/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs
@@ -27,11 +27,11 @@ fn successors() {
     let graph = create_graph();
     assert_eq!(graph.successors(0), &[1]);
     assert_eq!(graph.successors(1), &[2, 3]);
-    assert_eq!(graph.successors(2), &[]);
+    assert_eq!(graph.successors(2), &[] as &[usize]);
     assert_eq!(graph.successors(3), &[4]);
-    assert_eq!(graph.successors(4), &[]);
+    assert_eq!(graph.successors(4), &[] as &[usize]);
     assert_eq!(graph.successors(5), &[1]);
-    assert_eq!(graph.successors(6), &[]);
+    assert_eq!(graph.successors(6), &[] as &[usize]);
 }
 
 #[test]
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index a94e52fdf..e373bd184 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -1,5 +1,5 @@
 //! Various data structures used by the Rust compiler. The intention
-//! is that code in here should be not be *specific* to rustc, so that
+//! is that code in here should not be *specific* to rustc, so that
 //! it can be easily unit tested and so forth.
 //!
 //! # Note
@@ -20,13 +20,15 @@
 #![feature(never_type)]
 #![feature(type_alias_impl_trait)]
 #![feature(new_uninit)]
-#![feature(once_cell)]
+#![feature(lazy_cell)]
 #![feature(rustc_attrs)]
 #![feature(negative_impls)]
 #![feature(test)]
 #![feature(thread_id_value)]
 #![feature(vec_into_raw_parts)]
 #![feature(get_mut_unchecked)]
+#![feature(lint_reasons)]
+#![feature(unwrap_infallible)]
 #![allow(rustc::default_hash_types)]
 #![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
@@ -50,6 +52,7 @@ pub fn cold_path<F: FnOnce() -> R, R>(f: F) -> R {
 pub mod base_n;
 pub mod binary_search_util;
 pub mod captures;
+pub mod flat_map_in_place;
 pub mod flock;
 pub mod functor;
 pub mod fx;
@@ -57,9 +60,7 @@ pub mod graph;
 pub mod intern;
 pub mod jobserver;
 pub mod macros;
-pub mod map_in_place;
 pub mod obligation_forest;
-pub mod owning_ref;
 pub mod sip128;
 pub mod small_c_str;
 pub mod small_str;
@@ -79,10 +80,10 @@ pub mod sync;
 pub mod tiny_list;
 pub mod transitive_relation;
 pub mod vec_linked_list;
-pub mod vec_map;
 pub mod work_queue;
 pub use atomic_ref::AtomicRef;
 pub mod frozen;
+pub mod owned_slice;
 pub mod sso;
 pub mod steal;
 pub mod tagged_ptr;
diff --git a/compiler/rustc_data_structures/src/memmap.rs b/compiler/rustc_data_structures/src/memmap.rs
index 3d44e17f3..ef37a606f 100644
--- a/compiler/rustc_data_structures/src/memmap.rs
+++ b/compiler/rustc_data_structures/src/memmap.rs
@@ -2,9 +2,7 @@ use std::fs::File;
 use std::io;
 use std::ops::{Deref, DerefMut};
 
-use crate::owning_ref::StableAddress;
-
-/// A trivial wrapper for [`memmap2::Mmap`] that implements [`StableAddress`].
+/// A trivial wrapper for [`memmap2::Mmap`] (or `Vec<u8>` on WASM).
 #[cfg(not(target_arch = "wasm32"))]
 pub struct Mmap(memmap2::Mmap);
 
@@ -42,16 +40,10 @@ impl Deref for Mmap {
 
 impl AsRef<[u8]> for Mmap {
     fn as_ref(&self) -> &[u8] {
-        &*self.0
+        &self.0
     }
 }
 
-// SAFETY: On architectures other than WASM, mmap is used as backing storage. The address of this
-// memory map is stable. On WASM, `Vec<u8>` is used as backing storage. The `Mmap` type doesn't
-// export any function that can cause the `Vec` to be re-allocated. As such the address of the
-// bytes inside this `Vec` is stable.
-unsafe impl StableAddress for Mmap {}
-
 #[cfg(not(target_arch = "wasm32"))]
 pub struct MmapMut(memmap2::MmapMut);
 
diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs
index 91abdaada..27a869eb7 100644
--- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs
+++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs
@@ -97,7 +97,17 @@ pub trait ObligationProcessor {
     type Error: Debug;
     type OUT: OutcomeTrait<Obligation = Self::Obligation, Error = Error<Self::Obligation, Self::Error>>;
 
-    fn needs_process_obligation(&self, obligation: &Self::Obligation) -> bool;
+    /// Implementations can provide a fast-path to obligation-processing
+    /// by counting the prefix of the passed iterator for which
+    /// `needs_process_obligation` would return false.
+    fn skippable_obligations<'a>(
+        &'a self,
+        _it: impl Iterator<Item = &'a Self::Obligation>,
+    ) -> usize {
+        0
+    }
+
+    fn needs_process_obligation(&self, _obligation: &Self::Obligation) -> bool;
 
     fn process_obligation(
         &mut self,
@@ -416,6 +426,10 @@ impl<O: ForestObligation> ObligationForest<O> {
         loop {
             let mut has_changed = false;
 
+            // This is the super fast path for cheap-to-check conditions.
+            let mut index =
+                processor.skippable_obligations(self.nodes.iter().map(|n| &n.obligation));
+
             // Note that the loop body can append new nodes, and those new nodes
             // will then be processed by subsequent iterations of the loop.
             //
@@ -424,9 +438,8 @@ impl<O: ForestObligation> ObligationForest<O> {
             // `for index in 0..self.nodes.len() { ... }` because the range would
             // be computed with the initial length, and we would miss the appended
             // nodes. Therefore we use a `while` loop.
-            let mut index = 0;
             while let Some(node) = self.nodes.get_mut(index) {
-                // This test is extremely hot.
+                // This is the moderately fast path when the prefix skipping above didn't work out.
                 if node.state.get() != NodeState::Pending
                     || !processor.needs_process_obligation(&node.obligation)
                 {
diff --git a/compiler/rustc_data_structures/src/owned_slice.rs b/compiler/rustc_data_structures/src/owned_slice.rs
new file mode 100644
index 000000000..048401f66
--- /dev/null
+++ b/compiler/rustc_data_structures/src/owned_slice.rs
@@ -0,0 +1,118 @@
+use std::{borrow::Borrow, ops::Deref};
+
+// Use our fake Send/Sync traits when on not parallel compiler,
+// so that `OwnedSlice` only implements/requires Send/Sync
+// for parallel compiler builds.
+use crate::sync::{Send, Sync};
+
+/// An owned slice.
+///
+/// This is similar to `Box<[u8]>` but allows slicing and using anything as the
+/// backing buffer.
+///
+/// See [`slice_owned`] for `OwnedSlice` construction and examples.
+///
+/// ---------------------------------------------------------------------------
+///
+/// This is essentially a replacement for `owning_ref` which is a lot simpler
+/// and even sound! 🌸
+pub struct OwnedSlice {
+    /// This is conceptually a `&'self.owner [u8]`.
+    bytes: *const [u8],
+
+    // +---------------------------------------+
+    // | We expect `dead_code` lint here,      |
+    // | because we don't want to accidentally |
+    // | touch the owner — otherwise the owner |
+    // | could invalidate out `bytes` pointer  |
+    // |                                       |
+    // | so be quiet                           |
+    // +----+  +-------------------------------+
+    //       \/
+    //      ⊂(´･◡･⊂ )∘˚˳° (I am the phantom remnant of #97770)
+    #[expect(dead_code)]
+    owner: Box<dyn Send + Sync>,
+}
+
+/// Makes an [`OwnedSlice`] out of an `owner` and a `slicer` function.
+///
+/// ## Examples
+///
+/// ```rust
+/// # use rustc_data_structures::owned_slice::{OwnedSlice, slice_owned};
+/// let vec = vec![1, 2, 3, 4];
+///
+/// // Identical to slicing via `&v[1..3]` but produces an owned slice
+/// let slice: OwnedSlice = slice_owned(vec, |v| &v[1..3]);
+/// assert_eq!(&*slice, [2, 3]);
+/// ```
+///
+/// ```rust
+/// # use rustc_data_structures::owned_slice::{OwnedSlice, slice_owned};
+/// # use std::ops::Deref;
+/// let vec = vec![1, 2, 3, 4];
+///
+/// // Identical to slicing via `&v[..]` but produces an owned slice
+/// let slice: OwnedSlice = slice_owned(vec, Deref::deref);
+/// assert_eq!(&*slice, [1, 2, 3, 4]);
+/// ```
+pub fn slice_owned<O, F>(owner: O, slicer: F) -> OwnedSlice
+where
+    O: Send + Sync + 'static,
+    F: FnOnce(&O) -> &[u8],
+{
+    try_slice_owned(owner, |x| Ok::<_, !>(slicer(x))).into_ok()
+}
+
+/// Makes an [`OwnedSlice`] out of an `owner` and a `slicer` function that can fail.
+///
+/// See [`slice_owned`] for the infallible version.
+pub fn try_slice_owned<O, F, E>(owner: O, slicer: F) -> Result<OwnedSlice, E>
+where
+    O: Send + Sync + 'static,
+    F: FnOnce(&O) -> Result<&[u8], E>,
+{
+    // We box the owner of the bytes, so it doesn't move.
+    //
+    // Since the owner does not move and we don't access it in any way
+    // before drop, there is nothing that can invalidate the bytes pointer.
+    //
+    // Thus, "extending" the lifetime of the reference returned from `F` is fine.
+    // We pretend that we pass it a reference that lives as long as the returned slice.
+    //
+    // N.B. the HRTB on the `slicer` is important — without it the caller could provide
+    // a short lived slice, unrelated to the owner.
+
+    let owner = Box::new(owner);
+    let bytes = slicer(&*owner)?;
+
+    Ok(OwnedSlice { bytes, owner })
+}
+
+impl Deref for OwnedSlice {
+    type Target = [u8];
+
+    #[inline]
+    fn deref(&self) -> &[u8] {
+        // Safety:
+        // `self.bytes` is valid per the construction in `slice_owned`
+        // (which is the only constructor)
+        unsafe { &*self.bytes }
+    }
+}
+
+impl Borrow<[u8]> for OwnedSlice {
+    #[inline]
+    fn borrow(&self) -> &[u8] {
+        self
+    }
+}
+
+// Safety: `OwnedSlice` is conceptually `(&'self.1 [u8], Box<dyn Send + Sync>)`, which is `Send`
+unsafe impl Send for OwnedSlice {}
+
+// Safety: `OwnedSlice` is conceptually `(&'self.1 [u8], Box<dyn Send + Sync>)`, which is `Sync`
+unsafe impl Sync for OwnedSlice {}
+
+#[cfg(test)]
+mod tests;
diff --git a/compiler/rustc_data_structures/src/owned_slice/tests.rs b/compiler/rustc_data_structures/src/owned_slice/tests.rs
new file mode 100644
index 000000000..e715fb553
--- /dev/null
+++ b/compiler/rustc_data_structures/src/owned_slice/tests.rs
@@ -0,0 +1,74 @@
+use std::{
+    ops::Deref,
+    sync::{
+        atomic::{self, AtomicBool},
+        Arc,
+    },
+};
+
+use crate::{
+    owned_slice::{slice_owned, try_slice_owned, OwnedSlice},
+    OnDrop,
+};
+
+#[test]
+fn smoke() {
+    let slice = slice_owned(vec![1, 2, 3, 4, 5, 6], Vec::as_slice);
+
+    assert_eq!(&*slice, [1, 2, 3, 4, 5, 6]);
+}
+
+#[test]
+fn static_storage() {
+    let slice = slice_owned(Box::new(String::from("what")), |_| b"bytes boo");
+
+    assert_eq!(&*slice, b"bytes boo");
+}
+
+#[test]
+fn slice_the_slice() {
+    let slice = slice_owned(vec![1, 2, 3, 4, 5, 6], Vec::as_slice);
+    let slice = slice_owned(slice, |s| &s[1..][..4]);
+    let slice = slice_owned(slice, |s| s);
+    let slice = slice_owned(slice, |s| &s[1..]);
+
+    assert_eq!(&*slice, &[1, 2, 3, 4, 5, 6][1..][..4][1..]);
+}
+
+#[test]
+fn try_and_fail() {
+    let res = try_slice_owned(vec![0], |v| v.get(12..).ok_or(()));
+
+    assert!(res.is_err());
+}
+
+#[test]
+fn boxed() {
+    // It's important that we don't cause UB because of `Box`'es uniqueness
+
+    let boxed: Box<[u8]> = vec![1, 1, 2, 3, 5, 8, 13, 21].into_boxed_slice();
+    let slice = slice_owned(boxed, Deref::deref);
+
+    assert_eq!(&*slice, [1, 1, 2, 3, 5, 8, 13, 21]);
+}
+
+#[test]
+fn drop_drops() {
+    let flag = Arc::new(AtomicBool::new(false));
+    let flag_prime = Arc::clone(&flag);
+    let d = OnDrop(move || flag_prime.store(true, atomic::Ordering::Relaxed));
+
+    let slice = slice_owned(d, |_| &[]);
+
+    assert_eq!(flag.load(atomic::Ordering::Relaxed), false);
+
+    drop(slice);
+
+    assert_eq!(flag.load(atomic::Ordering::Relaxed), true);
+}
+
+#[test]
+fn send_sync() {
+    crate::sync::assert_send::<OwnedSlice>();
+    crate::sync::assert_sync::<OwnedSlice>();
+}
diff --git a/compiler/rustc_data_structures/src/owning_ref/LICENSE b/compiler/rustc_data_structures/src/owning_ref/LICENSE
deleted file mode 100644
index dff72d1e4..000000000
--- a/compiler/rustc_data_structures/src/owning_ref/LICENSE
+++ /dev/null
@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2015 Marvin Löbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/compiler/rustc_data_structures/src/owning_ref/mod.rs b/compiler/rustc_data_structures/src/owning_ref/mod.rs
deleted file mode 100644
index d1d92b905..000000000
--- a/compiler/rustc_data_structures/src/owning_ref/mod.rs
+++ /dev/null
@@ -1,1211 +0,0 @@
-#![warn(missing_docs)]
-
-/*!
-# An owning reference.
-
-This crate provides the _owning reference_ types `OwningRef` and `OwningRefMut`
-that enables it to bundle a reference together with the owner of the data it points to.
-This allows moving and dropping of an `OwningRef` without needing to recreate the reference.
-
-This can sometimes be useful because Rust borrowing rules normally prevent
-moving a type that has been moved from. For example, this kind of code gets rejected:
-
-```compile_fail,E0515
-fn return_owned_and_referenced<'a>() -> (Vec<u8>, &'a [u8]) {
-    let v = vec![1, 2, 3, 4];
-    let s = &v[1..3];
-    (v, s)
-}
-```
-
-Even though, from a memory-layout point of view, this can be entirely safe
-if the new location of the vector still lives longer than the lifetime `'a`
-of the reference because the backing allocation of the vector does not change.
-
-This library enables this safe usage by keeping the owner and the reference
-bundled together in a wrapper type that ensure that lifetime constraint:
-
-```
-# use rustc_data_structures::owning_ref::OwningRef;
-# fn main() {
-fn return_owned_and_referenced() -> OwningRef<Vec<u8>, [u8]> {
-    let v = vec![1, 2, 3, 4];
-    let or = OwningRef::new(v);
-    let or = or.map(|v| &v[1..3]);
-    or
-}
-# }
-```
-
-It works by requiring owner types to dereference to stable memory locations
-and preventing mutable access to root containers, which in practice requires heap allocation
-as provided by `Box<T>`, `Rc<T>`, etc.
-
-Also provided are typedefs for common owner type combinations,
-which allow for less verbose type signatures.
-For example, `BoxRef<T>` instead of `OwningRef<Box<T>, T>`.
-
-The crate also provides the more advanced `OwningHandle` type,
-which allows more freedom in bundling a dependent handle object
-along with the data it depends on, at the cost of some unsafe needed in the API.
-See the documentation around `OwningHandle` for more details.
-
-# Examples
-
-## Basics
-
-```
-use rustc_data_structures::owning_ref::BoxRef;
-
-fn main() {
-    // Create an array owned by a Box.
-    let arr = Box::new([1, 2, 3, 4]) as Box<[i32]>;
-
-    // Transfer into a BoxRef.
-    let arr: BoxRef<[i32]> = BoxRef::new(arr);
-    assert_eq!(&*arr, &[1, 2, 3, 4]);
-
-    // We can slice the array without losing ownership or changing type.
-    let arr: BoxRef<[i32]> = arr.map(|arr| &arr[1..3]);
-    assert_eq!(&*arr, &[2, 3]);
-
-    // Also works for Arc, Rc, String and Vec!
-}
-```
-
-## Caching a reference to a struct field
-
-```
-use rustc_data_structures::owning_ref::BoxRef;
-
-fn main() {
-    struct Foo {
-        tag: u32,
-        x: u16,
-        y: u16,
-        z: u16,
-    }
-    let foo = Foo { tag: 1, x: 100, y: 200, z: 300 };
-
-    let or = BoxRef::new(Box::new(foo)).map(|foo| {
-        match foo.tag {
-            0 => &foo.x,
-            1 => &foo.y,
-            2 => &foo.z,
-            _ => panic!(),
-        }
-    });
-
-    assert_eq!(*or, 200);
-}
-```
-
-## Caching a reference to an entry in a vector
-
-```
-use rustc_data_structures::owning_ref::VecRef;
-
-fn main() {
-    let v = VecRef::new(vec![1, 2, 3, 4, 5]).map(|v| &v[3]);
-    assert_eq!(*v, 4);
-}
-```
-
-## Caching a subslice of a String
-
-```
-use rustc_data_structures::owning_ref::StringRef;
-
-fn main() {
-    let s = StringRef::new("hello world".to_owned())
-        .map(|s| s.split(' ').nth(1).unwrap());
-
-    assert_eq!(&*s, "world");
-}
-```
-
-## Reference counted slices that share ownership of the backing storage
-
-```
-use rustc_data_structures::owning_ref::RcRef;
-use std::rc::Rc;
-
-fn main() {
-    let rc: RcRef<[i32]> = RcRef::new(Rc::new([1, 2, 3, 4]) as Rc<[i32]>);
-    assert_eq!(&*rc, &[1, 2, 3, 4]);
-
-    let rc_a: RcRef<[i32]> = rc.clone().map(|s| &s[0..2]);
-    let rc_b = rc.clone().map(|s| &s[1..3]);
-    let rc_c = rc.clone().map(|s| &s[2..4]);
-    assert_eq!(&*rc_a, &[1, 2]);
-    assert_eq!(&*rc_b, &[2, 3]);
-    assert_eq!(&*rc_c, &[3, 4]);
-
-    let rc_c_a = rc_c.clone().map(|s| &s[1]);
-    assert_eq!(&*rc_c_a, &4);
-}
-```
-
-## Atomic reference counted slices that share ownership of the backing storage
-
-```
-use rustc_data_structures::owning_ref::ArcRef;
-use std::sync::Arc;
-
-fn main() {
-    use std::thread;
-
-    fn par_sum(rc: ArcRef<[i32]>) -> i32 {
-        if rc.len() == 0 {
-            return 0;
-        } else if rc.len() == 1 {
-            return rc[0];
-        }
-        let mid = rc.len() / 2;
-        let left = rc.clone().map(|s| &s[..mid]);
-        let right = rc.map(|s| &s[mid..]);
-
-        let left = thread::spawn(move || par_sum(left));
-        let right = thread::spawn(move || par_sum(right));
-
-        left.join().unwrap() + right.join().unwrap()
-    }
-
-    let rc: Arc<[i32]> = Arc::new([1, 2, 3, 4]);
-    let rc: ArcRef<[i32]> = rc.into();
-
-    assert_eq!(par_sum(rc), 10);
-}
-```
-
-## References into RAII locks
-
-```
-use rustc_data_structures::owning_ref::RefRef;
-use std::cell::{RefCell, Ref};
-
-fn main() {
-    let refcell = RefCell::new((1, 2, 3, 4));
-    // Also works with Mutex and RwLock
-
-    let refref = {
-        let refref = RefRef::new(refcell.borrow()).map(|x| &x.3);
-        assert_eq!(*refref, 4);
-
-        // We move the RAII lock and the reference to one of
-        // the subfields in the data it guards here:
-        refref
-    };
-
-    assert_eq!(*refref, 4);
-
-    drop(refref);
-
-    assert_eq!(*refcell.borrow(), (1, 2, 3, 4));
-}
-```
-
-## Mutable reference
-
-When the owned container implements `DerefMut`, it is also possible to make
-a _mutable owning reference_. (e.g., with `Box`, `RefMut`, `MutexGuard`)
-
-```
-use rustc_data_structures::owning_ref::RefMutRefMut;
-use std::cell::{RefCell, RefMut};
-
-fn main() {
-    let refcell = RefCell::new((1, 2, 3, 4));
-
-    let mut refmut_refmut = {
-        let mut refmut_refmut = RefMutRefMut::new(refcell.borrow_mut()).map_mut(|x| &mut x.3);
-        assert_eq!(*refmut_refmut, 4);
-        *refmut_refmut *= 2;
-
-        refmut_refmut
-    };
-
-    assert_eq!(*refmut_refmut, 8);
-    *refmut_refmut *= 2;
-
-    drop(refmut_refmut);
-
-    assert_eq!(*refcell.borrow(), (1, 2, 3, 16));
-}
-```
-*/
-
-pub use stable_deref_trait::{
-    CloneStableDeref as CloneStableAddress, StableDeref as StableAddress,
-};
-use std::mem;
-
-/// An owning reference.
-///
-/// This wraps an owner `O` and a reference `&T` pointing
-/// at something reachable from `O::Target` while keeping
-/// the ability to move `self` around.
-///
-/// The owner is usually a pointer that points at some base type.
-///
-/// For more details and examples, see the module and method docs.
-pub struct OwningRef<O, T: ?Sized> {
-    owner: O,
-    reference: *const T,
-}
-
-/// An mutable owning reference.
-///
-/// This wraps an owner `O` and a reference `&mut T` pointing
-/// at something reachable from `O::Target` while keeping
-/// the ability to move `self` around.
-///
-/// The owner is usually a pointer that points at some base type.
-///
-/// For more details and examples, see the module and method docs.
-pub struct OwningRefMut<O, T: ?Sized> {
-    owner: O,
-    reference: *mut T,
-}
-
-/// Helper trait for an erased concrete type an owner dereferences to.
-/// This is used in form of a trait object for keeping
-/// something around to (virtually) call the destructor.
-pub trait Erased {}
-impl<T> Erased for T {}
-
-/// Helper trait for erasing the concrete type of what an owner dereferences to,
-/// for example `Box<T> -> Box<Erased>`. This would be unneeded with
-/// higher kinded types support in the language.
-#[allow(unused_lifetimes)]
-pub unsafe trait IntoErased<'a> {
-    /// Owner with the dereference type substituted to `Erased`.
-    type Erased;
-    /// Performs the type erasure.
-    fn into_erased(self) -> Self::Erased;
-}
-
-/// Helper trait for erasing the concrete type of what an owner dereferences to,
-/// for example `Box<T> -> Box<Erased + Send>`. This would be unneeded with
-/// higher kinded types support in the language.
-#[allow(unused_lifetimes)]
-pub unsafe trait IntoErasedSend<'a> {
-    /// Owner with the dereference type substituted to `Erased + Send`.
-    type Erased: Send;
-    /// Performs the type erasure.
-    fn into_erased_send(self) -> Self::Erased;
-}
-
-/// Helper trait for erasing the concrete type of what an owner dereferences to,
-/// for example `Box<T> -> Box<Erased + Send + Sync>`. This would be unneeded with
-/// higher kinded types support in the language.
-#[allow(unused_lifetimes)]
-pub unsafe trait IntoErasedSendSync<'a> {
-    /// Owner with the dereference type substituted to `Erased + Send + Sync`.
-    type Erased: Send + Sync;
-    /// Performs the type erasure.
-    fn into_erased_send_sync(self) -> Self::Erased;
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// OwningRef
-/////////////////////////////////////////////////////////////////////////////
-
-impl<O, T: ?Sized> OwningRef<O, T> {
-    /// Creates a new owning reference from an owner
-    /// initialized to the direct dereference of it.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::OwningRef;
-    ///
-    /// fn main() {
-    ///     let owning_ref = OwningRef::new(Box::new(42));
-    ///     assert_eq!(*owning_ref, 42);
-    /// }
-    /// ```
-    pub fn new(o: O) -> Self
-    where
-        O: StableAddress,
-        O: Deref<Target = T>,
-    {
-        OwningRef { reference: &*o, owner: o }
-    }
-
-    /// Like `new`, but doesn’t require `O` to implement the `StableAddress` trait.
-    /// Instead, the caller is responsible to make the same promises as implementing the trait.
-    ///
-    /// This is useful for cases where coherence rules prevents implementing the trait
-    /// without adding a dependency to this crate in a third-party library.
-    pub unsafe fn new_assert_stable_address(o: O) -> Self
-    where
-        O: Deref<Target = T>,
-    {
-        OwningRef { reference: &*o, owner: o }
-    }
-
-    /// Converts `self` into a new owning reference that points at something reachable
-    /// from the previous one.
-    ///
-    /// This can be a reference to a field of `U`, something reachable from a field of
-    /// `U`, or even something unrelated with a `'static` lifetime.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::OwningRef;
-    ///
-    /// fn main() {
-    ///     let owning_ref = OwningRef::new(Box::new([1, 2, 3, 4]));
-    ///
-    ///     // create an owning reference that points at the
-    ///     // third element of the array.
-    ///     let owning_ref = owning_ref.map(|array| &array[2]);
-    ///     assert_eq!(*owning_ref, 3);
-    /// }
-    /// ```
-    pub fn map<F, U: ?Sized>(self, f: F) -> OwningRef<O, U>
-    where
-        O: StableAddress,
-        F: FnOnce(&T) -> &U,
-    {
-        OwningRef { reference: f(&self), owner: self.owner }
-    }
-
-    /// Tries to convert `self` into a new owning reference that points
-    /// at something reachable from the previous one.
-    ///
-    /// This can be a reference to a field of `U`, something reachable from a field of
-    /// `U`, or even something unrelated with a `'static` lifetime.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::OwningRef;
-    ///
-    /// fn main() {
-    ///     let owning_ref = OwningRef::new(Box::new([1, 2, 3, 4]));
-    ///
-    ///     // create an owning reference that points at the
-    ///     // third element of the array.
-    ///     let owning_ref = owning_ref.try_map(|array| {
-    ///         if array[2] == 3 { Ok(&array[2]) } else { Err(()) }
-    ///     });
-    ///     assert_eq!(*owning_ref.unwrap(), 3);
-    /// }
-    /// ```
-    pub fn try_map<F, U: ?Sized, E>(self, f: F) -> Result<OwningRef<O, U>, E>
-    where
-        O: StableAddress,
-        F: FnOnce(&T) -> Result<&U, E>,
-    {
-        Ok(OwningRef { reference: f(&self)?, owner: self.owner })
-    }
-
-    /// Converts `self` into a new owning reference with a different owner type.
-    ///
-    /// The new owner type needs to still contain the original owner in some way
-    /// so that the reference into it remains valid. This function is marked unsafe
-    /// because the user needs to manually uphold this guarantee.
-    pub unsafe fn map_owner<F, P>(self, f: F) -> OwningRef<P, T>
-    where
-        O: StableAddress,
-        P: StableAddress,
-        F: FnOnce(O) -> P,
-    {
-        OwningRef { reference: self.reference, owner: f(self.owner) }
-    }
-
-    /// Converts `self` into a new owning reference where the owner is wrapped
-    /// in an additional `Box<O>`.
-    ///
-    /// This can be used to safely erase the owner of any `OwningRef<O, T>`
-    /// to an `OwningRef<Box<Erased>, T>`.
-    pub fn map_owner_box(self) -> OwningRef<Box<O>, T> {
-        OwningRef { reference: self.reference, owner: Box::new(self.owner) }
-    }
-
-    /// Erases the concrete base type of the owner with a trait object.
-    ///
-    /// This allows mixing of owned references with different owner base types.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::{OwningRef, Erased};
-    ///
-    /// fn main() {
-    ///     // N.B., using the concrete types here for explicitness.
-    ///     // For less verbose code type aliases like `BoxRef` are provided.
-    ///
-    ///     let owning_ref_a: OwningRef<Box<[i32; 4]>, [i32; 4]>
-    ///         = OwningRef::new(Box::new([1, 2, 3, 4]));
-    ///
-    ///     let owning_ref_b: OwningRef<Box<Vec<(i32, bool)>>, Vec<(i32, bool)>>
-    ///         = OwningRef::new(Box::new(vec![(0, false), (1, true)]));
-    ///
-    ///     let owning_ref_a: OwningRef<Box<[i32; 4]>, i32>
-    ///         = owning_ref_a.map(|a| &a[0]);
-    ///
-    ///     let owning_ref_b: OwningRef<Box<Vec<(i32, bool)>>, i32>
-    ///         = owning_ref_b.map(|a| &a[1].0);
-    ///
-    ///     let owning_refs: [OwningRef<Box<dyn Erased>, i32>; 2]
-    ///         = [owning_ref_a.erase_owner(), owning_ref_b.erase_owner()];
-    ///
-    ///     assert_eq!(*owning_refs[0], 1);
-    ///     assert_eq!(*owning_refs[1], 1);
-    /// }
-    /// ```
-    pub fn erase_owner<'a>(self) -> OwningRef<O::Erased, T>
-    where
-        O: IntoErased<'a>,
-    {
-        OwningRef { reference: self.reference, owner: self.owner.into_erased() }
-    }
-
-    /// Erases the concrete base type of the owner with a trait object which implements `Send`.
-    ///
-    /// This allows mixing of owned references with different owner base types.
-    pub fn erase_send_owner<'a>(self) -> OwningRef<O::Erased, T>
-    where
-        O: IntoErasedSend<'a>,
-    {
-        OwningRef { reference: self.reference, owner: self.owner.into_erased_send() }
-    }
-
-    /// Erases the concrete base type of the owner with a trait object
-    /// which implements `Send` and `Sync`.
-    ///
-    /// This allows mixing of owned references with different owner base types.
-    pub fn erase_send_sync_owner<'a>(self) -> OwningRef<O::Erased, T>
-    where
-        O: IntoErasedSendSync<'a>,
-    {
-        OwningRef { reference: self.reference, owner: self.owner.into_erased_send_sync() }
-    }
-
-    // UNIMPLEMENTED: wrap_owner
-
-    // FIXME: Naming convention?
-    /// A getter for the underlying owner.
-    pub fn owner(&self) -> &O {
-        &self.owner
-    }
-
-    // FIXME: Naming convention?
-    /// Discards the reference and retrieves the owner.
-    pub fn into_inner(self) -> O {
-        self.owner
-    }
-}
-
-impl<O, T: ?Sized> OwningRefMut<O, T> {
-    /// Creates a new owning reference from an owner
-    /// initialized to the direct dereference of it.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::OwningRefMut;
-    ///
-    /// fn main() {
-    ///     let owning_ref_mut = OwningRefMut::new(Box::new(42));
-    ///     assert_eq!(*owning_ref_mut, 42);
-    /// }
-    /// ```
-    pub fn new(mut o: O) -> Self
-    where
-        O: StableAddress,
-        O: DerefMut<Target = T>,
-    {
-        OwningRefMut { reference: &mut *o, owner: o }
-    }
-
-    /// Like `new`, but doesn’t require `O` to implement the `StableAddress` trait.
-    /// Instead, the caller is responsible to make the same promises as implementing the trait.
-    ///
-    /// This is useful for cases where coherence rules prevents implementing the trait
-    /// without adding a dependency to this crate in a third-party library.
-    pub unsafe fn new_assert_stable_address(mut o: O) -> Self
-    where
-        O: DerefMut<Target = T>,
-    {
-        OwningRefMut { reference: &mut *o, owner: o }
-    }
-
-    /// Converts `self` into a new _shared_ owning reference that points at
-    /// something reachable from the previous one.
-    ///
-    /// This can be a reference to a field of `U`, something reachable from a field of
-    /// `U`, or even something unrelated with a `'static` lifetime.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::OwningRefMut;
-    ///
-    /// fn main() {
-    ///     let owning_ref_mut = OwningRefMut::new(Box::new([1, 2, 3, 4]));
-    ///
-    ///     // create an owning reference that points at the
-    ///     // third element of the array.
-    ///     let owning_ref = owning_ref_mut.map(|array| &array[2]);
-    ///     assert_eq!(*owning_ref, 3);
-    /// }
-    /// ```
-    pub fn map<F, U: ?Sized>(mut self, f: F) -> OwningRef<O, U>
-    where
-        O: StableAddress,
-        F: FnOnce(&mut T) -> &U,
-    {
-        OwningRef { reference: f(&mut self), owner: self.owner }
-    }
-
-    /// Converts `self` into a new _mutable_ owning reference that points at
-    /// something reachable from the previous one.
-    ///
-    /// This can be a reference to a field of `U`, something reachable from a field of
-    /// `U`, or even something unrelated with a `'static` lifetime.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::OwningRefMut;
-    ///
-    /// fn main() {
-    ///     let owning_ref_mut = OwningRefMut::new(Box::new([1, 2, 3, 4]));
-    ///
-    ///     // create an owning reference that points at the
-    ///     // third element of the array.
-    ///     let owning_ref_mut = owning_ref_mut.map_mut(|array| &mut array[2]);
-    ///     assert_eq!(*owning_ref_mut, 3);
-    /// }
-    /// ```
-    pub fn map_mut<F, U: ?Sized>(mut self, f: F) -> OwningRefMut<O, U>
-    where
-        O: StableAddress,
-        F: FnOnce(&mut T) -> &mut U,
-    {
-        OwningRefMut { reference: f(&mut self), owner: self.owner }
-    }
-
-    /// Tries to convert `self` into a new _shared_ owning reference that points
-    /// at something reachable from the previous one.
-    ///
-    /// This can be a reference to a field of `U`, something reachable from a field of
-    /// `U`, or even something unrelated with a `'static` lifetime.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::OwningRefMut;
-    ///
-    /// fn main() {
-    ///     let owning_ref_mut = OwningRefMut::new(Box::new([1, 2, 3, 4]));
-    ///
-    ///     // create an owning reference that points at the
-    ///     // third element of the array.
-    ///     let owning_ref = owning_ref_mut.try_map(|array| {
-    ///         if array[2] == 3 { Ok(&array[2]) } else { Err(()) }
-    ///     });
-    ///     assert_eq!(*owning_ref.unwrap(), 3);
-    /// }
-    /// ```
-    pub fn try_map<F, U: ?Sized, E>(mut self, f: F) -> Result<OwningRef<O, U>, E>
-    where
-        O: StableAddress,
-        F: FnOnce(&mut T) -> Result<&U, E>,
-    {
-        Ok(OwningRef { reference: f(&mut self)?, owner: self.owner })
-    }
-
-    /// Tries to convert `self` into a new _mutable_ owning reference that points
-    /// at something reachable from the previous one.
-    ///
-    /// This can be a reference to a field of `U`, something reachable from a field of
-    /// `U`, or even something unrelated with a `'static` lifetime.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::OwningRefMut;
-    ///
-    /// fn main() {
-    ///     let owning_ref_mut = OwningRefMut::new(Box::new([1, 2, 3, 4]));
-    ///
-    ///     // create an owning reference that points at the
-    ///     // third element of the array.
-    ///     let owning_ref_mut = owning_ref_mut.try_map_mut(|array| {
-    ///         if array[2] == 3 { Ok(&mut array[2]) } else { Err(()) }
-    ///     });
-    ///     assert_eq!(*owning_ref_mut.unwrap(), 3);
-    /// }
-    /// ```
-    pub fn try_map_mut<F, U: ?Sized, E>(mut self, f: F) -> Result<OwningRefMut<O, U>, E>
-    where
-        O: StableAddress,
-        F: FnOnce(&mut T) -> Result<&mut U, E>,
-    {
-        Ok(OwningRefMut { reference: f(&mut self)?, owner: self.owner })
-    }
-
-    /// Converts `self` into a new owning reference with a different owner type.
-    ///
-    /// The new owner type needs to still contain the original owner in some way
-    /// so that the reference into it remains valid. This function is marked unsafe
-    /// because the user needs to manually uphold this guarantee.
-    pub unsafe fn map_owner<F, P>(self, f: F) -> OwningRefMut<P, T>
-    where
-        O: StableAddress,
-        P: StableAddress,
-        F: FnOnce(O) -> P,
-    {
-        OwningRefMut { reference: self.reference, owner: f(self.owner) }
-    }
-
-    /// Converts `self` into a new owning reference where the owner is wrapped
-    /// in an additional `Box<O>`.
-    ///
-    /// This can be used to safely erase the owner of any `OwningRefMut<O, T>`
-    /// to an `OwningRefMut<Box<Erased>, T>`.
-    pub fn map_owner_box(self) -> OwningRefMut<Box<O>, T> {
-        OwningRefMut { reference: self.reference, owner: Box::new(self.owner) }
-    }
-
-    /// Erases the concrete base type of the owner with a trait object.
-    ///
-    /// This allows mixing of owned references with different owner base types.
-    ///
-    /// # Example
-    /// ```
-    /// use rustc_data_structures::owning_ref::{OwningRefMut, Erased};
-    ///
-    /// fn main() {
-    ///     // N.B., using the concrete types here for explicitness.
-    ///     // For less verbose code type aliases like `BoxRef` are provided.
-    ///
-    ///     let owning_ref_mut_a: OwningRefMut<Box<[i32; 4]>, [i32; 4]>
-    ///         = OwningRefMut::new(Box::new([1, 2, 3, 4]));
-    ///
-    ///     let owning_ref_mut_b: OwningRefMut<Box<Vec<(i32, bool)>>, Vec<(i32, bool)>>
-    ///         = OwningRefMut::new(Box::new(vec![(0, false), (1, true)]));
-    ///
-    ///     let owning_ref_mut_a: OwningRefMut<Box<[i32; 4]>, i32>
-    ///         = owning_ref_mut_a.map_mut(|a| &mut a[0]);
-    ///
-    ///     let owning_ref_mut_b: OwningRefMut<Box<Vec<(i32, bool)>>, i32>
-    ///         = owning_ref_mut_b.map_mut(|a| &mut a[1].0);
-    ///
-    ///     let owning_refs_mut: [OwningRefMut<Box<dyn Erased>, i32>; 2]
-    ///         = [owning_ref_mut_a.erase_owner(), owning_ref_mut_b.erase_owner()];
-    ///
-    ///     assert_eq!(*owning_refs_mut[0], 1);
-    ///     assert_eq!(*owning_refs_mut[1], 1);
-    /// }
-    /// ```
-    pub fn erase_owner<'a>(self) -> OwningRefMut<O::Erased, T>
-    where
-        O: IntoErased<'a>,
-    {
-        OwningRefMut { reference: self.reference, owner: self.owner.into_erased() }
-    }
-
-    // UNIMPLEMENTED: wrap_owner
-
-    // FIXME: Naming convention?
-    /// A getter for the underlying owner.
-    pub fn owner(&self) -> &O {
-        &self.owner
-    }
-
-    // FIXME: Naming convention?
-    /// Discards the reference and retrieves the owner.
-    pub fn into_inner(self) -> O {
-        self.owner
-    }
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// OwningHandle
-/////////////////////////////////////////////////////////////////////////////
-
-use std::ops::{Deref, DerefMut};
-
-/// `OwningHandle` is a complement to `OwningRef`. Where `OwningRef` allows
-/// consumers to pass around an owned object and a dependent reference,
-/// `OwningHandle` contains an owned object and a dependent _object_.
-///
-/// `OwningHandle` can encapsulate a `RefMut` along with its associated
-/// `RefCell`, or an `RwLockReadGuard` along with its associated `RwLock`.
-/// However, the API is completely generic and there are no restrictions on
-/// what types of owning and dependent objects may be used.
-///
-/// `OwningHandle` is created by passing an owner object (which dereferences
-/// to a stable address) along with a callback which receives a pointer to
-/// that stable location. The callback may then dereference the pointer and
-/// mint a dependent object, with the guarantee that the returned object will
-/// not outlive the referent of the pointer.
-///
-/// Since the callback needs to dereference a raw pointer, it requires `unsafe`
-/// code. To avoid forcing this unsafety on most callers, the `ToHandle` trait is
-/// implemented for common data structures. Types that implement `ToHandle` can
-/// be wrapped into an `OwningHandle` without passing a callback.
-pub struct OwningHandle<O, H>
-where
-    O: StableAddress,
-    H: Deref,
-{
-    handle: H,
-    _owner: O,
-}
-
-impl<O, H> Deref for OwningHandle<O, H>
-where
-    O: StableAddress,
-    H: Deref,
-{
-    type Target = H::Target;
-    fn deref(&self) -> &H::Target {
-        self.handle.deref()
-    }
-}
-
-unsafe impl<O, H> StableAddress for OwningHandle<O, H>
-where
-    O: StableAddress,
-    H: StableAddress,
-{
-}
-
-impl<O, H> DerefMut for OwningHandle<O, H>
-where
-    O: StableAddress,
-    H: DerefMut,
-{
-    fn deref_mut(&mut self) -> &mut H::Target {
-        self.handle.deref_mut()
-    }
-}
-
-/// Trait to implement the conversion of owner to handle for common types.
-pub trait ToHandle {
-    /// The type of handle to be encapsulated by the OwningHandle.
-    type Handle: Deref;
-
-    /// Given an appropriately-long-lived pointer to ourselves, create a
-    /// handle to be encapsulated by the `OwningHandle`.
-    unsafe fn to_handle(x: *const Self) -> Self::Handle;
-}
-
-/// Trait to implement the conversion of owner to mutable handle for common types.
-pub trait ToHandleMut {
-    /// The type of handle to be encapsulated by the OwningHandle.
-    type HandleMut: DerefMut;
-
-    /// Given an appropriately-long-lived pointer to ourselves, create a
-    /// mutable handle to be encapsulated by the `OwningHandle`.
-    unsafe fn to_handle_mut(x: *const Self) -> Self::HandleMut;
-}
-
-impl<O, H> OwningHandle<O, H>
-where
-    O: StableAddress<Target: ToHandle<Handle = H>>,
-    H: Deref,
-{
-    /// Creates a new `OwningHandle` for a type that implements `ToHandle`. For types
-    /// that don't implement `ToHandle`, callers may invoke `new_with_fn`, which accepts
-    /// a callback to perform the conversion.
-    pub fn new(o: O) -> Self {
-        OwningHandle::new_with_fn(o, |x| unsafe { O::Target::to_handle(x) })
-    }
-}
-
-impl<O, H> OwningHandle<O, H>
-where
-    O: StableAddress<Target: ToHandleMut<HandleMut = H>>,
-    H: DerefMut,
-{
-    /// Creates a new mutable `OwningHandle` for a type that implements `ToHandleMut`.
-    pub fn new_mut(o: O) -> Self {
-        OwningHandle::new_with_fn(o, |x| unsafe { O::Target::to_handle_mut(x) })
-    }
-}
-
-impl<O, H> OwningHandle<O, H>
-where
-    O: StableAddress,
-    H: Deref,
-{
-    /// Creates a new OwningHandle. The provided callback will be invoked with
-    /// a pointer to the object owned by `o`, and the returned value is stored
-    /// as the object to which this `OwningHandle` will forward `Deref` and
-    /// `DerefMut`.
-    pub fn new_with_fn<F>(o: O, f: F) -> Self
-    where
-        F: FnOnce(*const O::Target) -> H,
-    {
-        let h: H;
-        {
-            h = f(o.deref() as *const O::Target);
-        }
-
-        OwningHandle { handle: h, _owner: o }
-    }
-
-    /// Creates a new OwningHandle. The provided callback will be invoked with
-    /// a pointer to the object owned by `o`, and the returned value is stored
-    /// as the object to which this `OwningHandle` will forward `Deref` and
-    /// `DerefMut`.
-    pub fn try_new<F, E>(o: O, f: F) -> Result<Self, E>
-    where
-        F: FnOnce(*const O::Target) -> Result<H, E>,
-    {
-        let h: H;
-        {
-            h = f(o.deref() as *const O::Target)?;
-        }
-
-        Ok(OwningHandle { handle: h, _owner: o })
-    }
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// std traits
-/////////////////////////////////////////////////////////////////////////////
-
-use std::borrow::Borrow;
-use std::cmp::Ordering;
-use std::fmt::{self, Debug};
-use std::hash::{Hash, Hasher};
-
-impl<O, T: ?Sized> Deref for OwningRef<O, T> {
-    type Target = T;
-
-    fn deref(&self) -> &T {
-        unsafe { &*self.reference }
-    }
-}
-
-impl<O, T: ?Sized> Deref for OwningRefMut<O, T> {
-    type Target = T;
-
-    fn deref(&self) -> &T {
-        unsafe { &*self.reference }
-    }
-}
-
-impl<O, T: ?Sized> DerefMut for OwningRefMut<O, T> {
-    fn deref_mut(&mut self) -> &mut T {
-        unsafe { &mut *self.reference }
-    }
-}
-
-unsafe impl<O, T: ?Sized> StableAddress for OwningRef<O, T> {}
-
-impl<O, T: ?Sized> AsRef<T> for OwningRef<O, T> {
-    fn as_ref(&self) -> &T {
-        self
-    }
-}
-
-impl<O, T: ?Sized> AsRef<T> for OwningRefMut<O, T> {
-    fn as_ref(&self) -> &T {
-        self
-    }
-}
-
-impl<O, T: ?Sized> AsMut<T> for OwningRefMut<O, T> {
-    fn as_mut(&mut self) -> &mut T {
-        self
-    }
-}
-
-impl<O, T: ?Sized> Borrow<T> for OwningRef<O, T> {
-    fn borrow(&self) -> &T {
-        self
-    }
-}
-
-impl<O, T: ?Sized> From<O> for OwningRef<O, T>
-where
-    O: StableAddress,
-    O: Deref<Target = T>,
-{
-    fn from(owner: O) -> Self {
-        OwningRef::new(owner)
-    }
-}
-
-impl<O, T: ?Sized> From<O> for OwningRefMut<O, T>
-where
-    O: StableAddress,
-    O: DerefMut<Target = T>,
-{
-    fn from(owner: O) -> Self {
-        OwningRefMut::new(owner)
-    }
-}
-
-impl<O, T: ?Sized> From<OwningRefMut<O, T>> for OwningRef<O, T>
-where
-    O: StableAddress,
-    O: DerefMut<Target = T>,
-{
-    fn from(other: OwningRefMut<O, T>) -> Self {
-        OwningRef { owner: other.owner, reference: other.reference }
-    }
-}
-
-// ^ FIXME: Is an Into impl for calling into_inner() possible as well?
-
-impl<O, T: ?Sized> Debug for OwningRef<O, T>
-where
-    O: Debug,
-    T: Debug,
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "OwningRef {{ owner: {:?}, reference: {:?} }}", self.owner(), &**self)
-    }
-}
-
-impl<O, T: ?Sized> Debug for OwningRefMut<O, T>
-where
-    O: Debug,
-    T: Debug,
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "OwningRefMut {{ owner: {:?}, reference: {:?} }}", self.owner(), &**self)
-    }
-}
-
-impl<O, T: ?Sized> Clone for OwningRef<O, T>
-where
-    O: CloneStableAddress,
-{
-    fn clone(&self) -> Self {
-        OwningRef { owner: self.owner.clone(), reference: self.reference }
-    }
-}
-
-unsafe impl<O, T: ?Sized> CloneStableAddress for OwningRef<O, T> where O: CloneStableAddress {}
-
-unsafe impl<O, T: ?Sized> Send for OwningRef<O, T>
-where
-    O: Send,
-    for<'a> &'a T: Send,
-{
-}
-unsafe impl<O, T: ?Sized> Sync for OwningRef<O, T>
-where
-    O: Sync,
-    for<'a> &'a T: Sync,
-{
-}
-
-unsafe impl<O, T: ?Sized> Send for OwningRefMut<O, T>
-where
-    O: Send,
-    for<'a> &'a mut T: Send,
-{
-}
-unsafe impl<O, T: ?Sized> Sync for OwningRefMut<O, T>
-where
-    O: Sync,
-    for<'a> &'a mut T: Sync,
-{
-}
-
-impl Debug for dyn Erased {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "<Erased>",)
-    }
-}
-
-impl<O, T: ?Sized> PartialEq for OwningRef<O, T>
-where
-    T: PartialEq,
-{
-    fn eq(&self, other: &Self) -> bool {
-        self.deref().eq(other.deref())
-    }
-}
-
-impl<O, T: ?Sized> Eq for OwningRef<O, T> where T: Eq {}
-
-impl<O, T: ?Sized> PartialOrd for OwningRef<O, T>
-where
-    T: PartialOrd,
-{
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        self.deref().partial_cmp(other.deref())
-    }
-}
-
-impl<O, T: ?Sized> Ord for OwningRef<O, T>
-where
-    T: Ord,
-{
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.deref().cmp(other.deref())
-    }
-}
-
-impl<O, T: ?Sized> Hash for OwningRef<O, T>
-where
-    T: Hash,
-{
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.deref().hash(state);
-    }
-}
-
-impl<O, T: ?Sized> PartialEq for OwningRefMut<O, T>
-where
-    T: PartialEq,
-{
-    fn eq(&self, other: &Self) -> bool {
-        self.deref().eq(other.deref())
-    }
-}
-
-impl<O, T: ?Sized> Eq for OwningRefMut<O, T> where T: Eq {}
-
-impl<O, T: ?Sized> PartialOrd for OwningRefMut<O, T>
-where
-    T: PartialOrd,
-{
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        self.deref().partial_cmp(other.deref())
-    }
-}
-
-impl<O, T: ?Sized> Ord for OwningRefMut<O, T>
-where
-    T: Ord,
-{
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.deref().cmp(other.deref())
-    }
-}
-
-impl<O, T: ?Sized> Hash for OwningRefMut<O, T>
-where
-    T: Hash,
-{
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.deref().hash(state);
-    }
-}
-
-/////////////////////////////////////////////////////////////////////////////
-// std types integration and convenience type defs
-/////////////////////////////////////////////////////////////////////////////
-
-use std::cell::{Ref, RefCell, RefMut};
-use std::rc::Rc;
-use std::sync::Arc;
-use std::sync::{MutexGuard, RwLockReadGuard, RwLockWriteGuard};
-
-impl<T: 'static> ToHandle for RefCell<T> {
-    type Handle = Ref<'static, T>;
-    unsafe fn to_handle(x: *const Self) -> Self::Handle {
-        (*x).borrow()
-    }
-}
-
-impl<T: 'static> ToHandleMut for RefCell<T> {
-    type HandleMut = RefMut<'static, T>;
-    unsafe fn to_handle_mut(x: *const Self) -> Self::HandleMut {
-        (*x).borrow_mut()
-    }
-}
-
-// N.B., implementing ToHandle{,Mut} for Mutex and RwLock requires a decision
-// about which handle creation to use (i.e., read() vs try_read()) as well as
-// what to do with error results.
-
-/// Typedef of an owning reference that uses a `Box` as the owner.
-pub type BoxRef<T, U = T> = OwningRef<Box<T>, U>;
-/// Typedef of an owning reference that uses a `Vec` as the owner.
-pub type VecRef<T, U = T> = OwningRef<Vec<T>, U>;
-/// Typedef of an owning reference that uses a `String` as the owner.
-pub type StringRef = OwningRef<String, str>;
-
-/// Typedef of an owning reference that uses an `Rc` as the owner.
-pub type RcRef<T, U = T> = OwningRef<Rc<T>, U>;
-/// Typedef of an owning reference that uses an `Arc` as the owner.
-pub type ArcRef<T, U = T> = OwningRef<Arc<T>, U>;
-
-/// Typedef of an owning reference that uses a `Ref` as the owner.
-pub type RefRef<'a, T, U = T> = OwningRef<Ref<'a, T>, U>;
-/// Typedef of an owning reference that uses a `RefMut` as the owner.
-pub type RefMutRef<'a, T, U = T> = OwningRef<RefMut<'a, T>, U>;
-/// Typedef of an owning reference that uses a `MutexGuard` as the owner.
-pub type MutexGuardRef<'a, T, U = T> = OwningRef<MutexGuard<'a, T>, U>;
-/// Typedef of an owning reference that uses an `RwLockReadGuard` as the owner.
-pub type RwLockReadGuardRef<'a, T, U = T> = OwningRef<RwLockReadGuard<'a, T>, U>;
-/// Typedef of an owning reference that uses an `RwLockWriteGuard` as the owner.
-pub type RwLockWriteGuardRef<'a, T, U = T> = OwningRef<RwLockWriteGuard<'a, T>, U>;
-
-/// Typedef of a mutable owning reference that uses a `Box` as the owner.
-pub type BoxRefMut<T, U = T> = OwningRefMut<Box<T>, U>;
-/// Typedef of a mutable owning reference that uses a `Vec` as the owner.
-pub type VecRefMut<T, U = T> = OwningRefMut<Vec<T>, U>;
-/// Typedef of a mutable owning reference that uses a `String` as the owner.
-pub type StringRefMut = OwningRefMut<String, str>;
-
-/// Typedef of a mutable owning reference that uses a `RefMut` as the owner.
-pub type RefMutRefMut<'a, T, U = T> = OwningRefMut<RefMut<'a, T>, U>;
-/// Typedef of a mutable owning reference that uses a `MutexGuard` as the owner.
-pub type MutexGuardRefMut<'a, T, U = T> = OwningRefMut<MutexGuard<'a, T>, U>;
-/// Typedef of a mutable owning reference that uses an `RwLockWriteGuard` as the owner.
-pub type RwLockWriteGuardRefMut<'a, T, U = T> = OwningRef<RwLockWriteGuard<'a, T>, U>;
-
-unsafe impl<'a, T: 'a> IntoErased<'a> for Box<T> {
-    type Erased = Box<dyn Erased + 'a>;
-    fn into_erased(self) -> Self::Erased {
-        self
-    }
-}
-unsafe impl<'a, T: 'a> IntoErased<'a> for Rc<T> {
-    type Erased = Rc<dyn Erased + 'a>;
-    fn into_erased(self) -> Self::Erased {
-        self
-    }
-}
-unsafe impl<'a, T: 'a> IntoErased<'a> for Arc<T> {
-    type Erased = Arc<dyn Erased + 'a>;
-    fn into_erased(self) -> Self::Erased {
-        self
-    }
-}
-
-unsafe impl<'a, T: Send + 'a> IntoErasedSend<'a> for Box<T> {
-    type Erased = Box<dyn Erased + Send + 'a>;
-    fn into_erased_send(self) -> Self::Erased {
-        self
-    }
-}
-
-unsafe impl<'a, T: Send + 'a> IntoErasedSendSync<'a> for Box<T> {
-    type Erased = Box<dyn Erased + Sync + Send + 'a>;
-    fn into_erased_send_sync(self) -> Self::Erased {
-        let result: Box<dyn Erased + Send + 'a> = self;
-        // This is safe since Erased can always implement Sync
-        // Only the destructor is available and it takes &mut self
-        unsafe { mem::transmute(result) }
-    }
-}
-
-unsafe impl<'a, T: Send + Sync + 'a> IntoErasedSendSync<'a> for Arc<T> {
-    type Erased = Arc<dyn Erased + Send + Sync + 'a>;
-    fn into_erased_send_sync(self) -> Self::Erased {
-        self
-    }
-}
-
-/// Typedef of an owning reference that uses an erased `Box` as the owner.
-pub type ErasedBoxRef<U> = OwningRef<Box<dyn Erased>, U>;
-/// Typedef of an owning reference that uses an erased `Rc` as the owner.
-pub type ErasedRcRef<U> = OwningRef<Rc<dyn Erased>, U>;
-/// Typedef of an owning reference that uses an erased `Arc` as the owner.
-pub type ErasedArcRef<U> = OwningRef<Arc<dyn Erased>, U>;
-
-/// Typedef of a mutable owning reference that uses an erased `Box` as the owner.
-pub type ErasedBoxRefMut<U> = OwningRefMut<Box<dyn Erased>, U>;
-
-#[cfg(test)]
-mod tests;
diff --git a/compiler/rustc_data_structures/src/owning_ref/tests.rs b/compiler/rustc_data_structures/src/owning_ref/tests.rs
deleted file mode 100644
index a9b187c4c..000000000
--- a/compiler/rustc_data_structures/src/owning_ref/tests.rs
+++ /dev/null
@@ -1,711 +0,0 @@
-// FIXME: owning_ref is not sound under stacked borrows. Preferably, get rid of it.
-#[cfg(not(miri))]
-mod owning_ref {
-    use super::super::OwningRef;
-    use super::super::{BoxRef, Erased, ErasedBoxRef, RcRef};
-    use std::cmp::Ordering;
-    use std::collections::hash_map::DefaultHasher;
-    use std::collections::HashMap;
-    use std::hash::{Hash, Hasher};
-    use std::rc::Rc;
-
-    #[derive(Debug, PartialEq)]
-    struct Example(u32, String, [u8; 3]);
-    fn example() -> Example {
-        Example(42, "hello world".to_string(), [1, 2, 3])
-    }
-
-    #[test]
-    fn new_deref() {
-        let or: OwningRef<Box<()>, ()> = OwningRef::new(Box::new(()));
-        assert_eq!(&*or, &());
-    }
-
-    #[test]
-    fn into() {
-        let or: OwningRef<Box<()>, ()> = Box::new(()).into();
-        assert_eq!(&*or, &());
-    }
-
-    #[test]
-    fn map_offset_ref() {
-        let or: BoxRef<Example> = Box::new(example()).into();
-        let or: BoxRef<_, u32> = or.map(|x| &x.0);
-        assert_eq!(&*or, &42);
-
-        let or: BoxRef<Example> = Box::new(example()).into();
-        let or: BoxRef<_, u8> = or.map(|x| &x.2[1]);
-        assert_eq!(&*or, &2);
-    }
-
-    #[test]
-    fn map_heap_ref() {
-        let or: BoxRef<Example> = Box::new(example()).into();
-        let or: BoxRef<_, str> = or.map(|x| &x.1[..5]);
-        assert_eq!(&*or, "hello");
-    }
-
-    #[test]
-    fn map_static_ref() {
-        let or: BoxRef<()> = Box::new(()).into();
-        let or: BoxRef<_, str> = or.map(|_| "hello");
-        assert_eq!(&*or, "hello");
-    }
-
-    #[test]
-    fn map_chained() {
-        let or: BoxRef<String> = Box::new(example().1).into();
-        let or: BoxRef<_, str> = or.map(|x| &x[1..5]);
-        let or: BoxRef<_, str> = or.map(|x| &x[..2]);
-        assert_eq!(&*or, "el");
-    }
-
-    #[test]
-    fn map_chained_inference() {
-        let or = BoxRef::new(Box::new(example().1)).map(|x| &x[..5]).map(|x| &x[1..3]);
-        assert_eq!(&*or, "el");
-    }
-
-    #[test]
-    fn owner() {
-        let or: BoxRef<String> = Box::new(example().1).into();
-        let or = or.map(|x| &x[..5]);
-        assert_eq!(&*or, "hello");
-        assert_eq!(&**or.owner(), "hello world");
-    }
-
-    #[test]
-    fn into_inner() {
-        let or: BoxRef<String> = Box::new(example().1).into();
-        let or = or.map(|x| &x[..5]);
-        assert_eq!(&*or, "hello");
-        let s = *or.into_inner();
-        assert_eq!(&s, "hello world");
-    }
-
-    #[test]
-    fn fmt_debug() {
-        let or: BoxRef<String> = Box::new(example().1).into();
-        let or = or.map(|x| &x[..5]);
-        let s = format!("{:?}", or);
-        assert_eq!(&s, "OwningRef { owner: \"hello world\", reference: \"hello\" }");
-    }
-
-    #[test]
-    fn erased_owner() {
-        let o1: BoxRef<Example, str> = BoxRef::new(Box::new(example())).map(|x| &x.1[..]);
-
-        let o2: BoxRef<String, str> = BoxRef::new(Box::new(example().1)).map(|x| &x[..]);
-
-        let os: Vec<ErasedBoxRef<str>> = vec![o1.erase_owner(), o2.erase_owner()];
-        assert!(os.iter().all(|e| &e[..] == "hello world"));
-    }
-
-    #[test]
-    fn raii_locks() {
-        use super::super::{MutexGuardRef, RwLockReadGuardRef, RwLockWriteGuardRef};
-        use super::super::{RefMutRef, RefRef};
-        use std::cell::RefCell;
-        use std::sync::{Mutex, RwLock};
-
-        {
-            let a = RefCell::new(1);
-            let a = {
-                let a = RefRef::new(a.borrow());
-                assert_eq!(*a, 1);
-                a
-            };
-            assert_eq!(*a, 1);
-            drop(a);
-        }
-        {
-            let a = RefCell::new(1);
-            let a = {
-                let a = RefMutRef::new(a.borrow_mut());
-                assert_eq!(*a, 1);
-                a
-            };
-            assert_eq!(*a, 1);
-            drop(a);
-        }
-        {
-            let a = Mutex::new(1);
-            let a = {
-                let a = MutexGuardRef::new(a.lock().unwrap());
-                assert_eq!(*a, 1);
-                a
-            };
-            assert_eq!(*a, 1);
-            drop(a);
-        }
-        {
-            let a = RwLock::new(1);
-            let a = {
-                let a = RwLockReadGuardRef::new(a.read().unwrap());
-                assert_eq!(*a, 1);
-                a
-            };
-            assert_eq!(*a, 1);
-            drop(a);
-        }
-        {
-            let a = RwLock::new(1);
-            let a = {
-                let a = RwLockWriteGuardRef::new(a.write().unwrap());
-                assert_eq!(*a, 1);
-                a
-            };
-            assert_eq!(*a, 1);
-            drop(a);
-        }
-    }
-
-    #[test]
-    fn eq() {
-        let or1: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice());
-        let or2: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice());
-        assert_eq!(or1.eq(&or2), true);
-    }
-
-    #[test]
-    fn cmp() {
-        let or1: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice());
-        let or2: BoxRef<[u8]> = BoxRef::new(vec![4, 5, 6].into_boxed_slice());
-        assert_eq!(or1.cmp(&or2), Ordering::Less);
-    }
-
-    #[test]
-    fn partial_cmp() {
-        let or1: BoxRef<[u8]> = BoxRef::new(vec![4, 5, 6].into_boxed_slice());
-        let or2: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice());
-        assert_eq!(or1.partial_cmp(&or2), Some(Ordering::Greater));
-    }
-
-    #[test]
-    fn hash() {
-        let mut h1 = DefaultHasher::new();
-        let mut h2 = DefaultHasher::new();
-
-        let or1: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice());
-        let or2: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice());
-
-        or1.hash(&mut h1);
-        or2.hash(&mut h2);
-
-        assert_eq!(h1.finish(), h2.finish());
-    }
-
-    #[test]
-    fn borrow() {
-        let mut hash = HashMap::new();
-        let key = RcRef::<String>::new(Rc::new("foo-bar".to_string())).map(|s| &s[..]);
-
-        hash.insert(key.clone().map(|s| &s[..3]), 42);
-        hash.insert(key.clone().map(|s| &s[4..]), 23);
-
-        assert_eq!(hash.get("foo"), Some(&42));
-        assert_eq!(hash.get("bar"), Some(&23));
-    }
-
-    #[test]
-    fn total_erase() {
-        let a: OwningRef<Vec<u8>, [u8]> = OwningRef::new(vec![]).map(|x| &x[..]);
-        let b: OwningRef<Box<[u8]>, [u8]> =
-            OwningRef::new(vec![].into_boxed_slice()).map(|x| &x[..]);
-
-        let c: OwningRef<Rc<Vec<u8>>, [u8]> = unsafe { a.map_owner(Rc::new) };
-        let d: OwningRef<Rc<Box<[u8]>>, [u8]> = unsafe { b.map_owner(Rc::new) };
-
-        let e: OwningRef<Rc<dyn Erased>, [u8]> = c.erase_owner();
-        let f: OwningRef<Rc<dyn Erased>, [u8]> = d.erase_owner();
-
-        let _g = e.clone();
-        let _h = f.clone();
-    }
-
-    #[test]
-    fn total_erase_box() {
-        let a: OwningRef<Vec<u8>, [u8]> = OwningRef::new(vec![]).map(|x| &x[..]);
-        let b: OwningRef<Box<[u8]>, [u8]> =
-            OwningRef::new(vec![].into_boxed_slice()).map(|x| &x[..]);
-
-        let c: OwningRef<Box<Vec<u8>>, [u8]> = a.map_owner_box();
-        let d: OwningRef<Box<Box<[u8]>>, [u8]> = b.map_owner_box();
-
-        let _e: OwningRef<Box<dyn Erased>, [u8]> = c.erase_owner();
-        let _f: OwningRef<Box<dyn Erased>, [u8]> = d.erase_owner();
-    }
-
-    #[test]
-    fn try_map1() {
-        use std::any::Any;
-
-        let x = Box::new(123_i32);
-        let y: Box<dyn Any> = x;
-
-        assert!(OwningRef::new(y).try_map(|x| x.downcast_ref::<i32>().ok_or(())).is_ok());
-    }
-
-    #[test]
-    fn try_map2() {
-        use std::any::Any;
-
-        let x = Box::new(123_i32);
-        let y: Box<dyn Any> = x;
-
-        assert!(!OwningRef::new(y).try_map(|x| x.downcast_ref::<i32>().ok_or(())).is_err());
-    }
-}
-
-mod owning_handle {
-    use super::super::OwningHandle;
-    use super::super::RcRef;
-    use std::cell::RefCell;
-    use std::rc::Rc;
-    use std::sync::Arc;
-    use std::sync::RwLock;
-
-    #[test]
-    fn owning_handle() {
-        use std::cell::RefCell;
-        let cell = Rc::new(RefCell::new(2));
-        let cell_ref = RcRef::new(cell);
-        let mut handle =
-            OwningHandle::new_with_fn(cell_ref, |x| unsafe { x.as_ref() }.unwrap().borrow_mut());
-        assert_eq!(*handle, 2);
-        *handle = 3;
-        assert_eq!(*handle, 3);
-    }
-
-    #[test]
-    fn try_owning_handle_ok() {
-        use std::cell::RefCell;
-        let cell = Rc::new(RefCell::new(2));
-        let cell_ref = RcRef::new(cell);
-        let mut handle = OwningHandle::try_new::<_, ()>(cell_ref, |x| {
-            Ok(unsafe { x.as_ref() }.unwrap().borrow_mut())
-        })
-        .unwrap();
-        assert_eq!(*handle, 2);
-        *handle = 3;
-        assert_eq!(*handle, 3);
-    }
-
-    #[test]
-    fn try_owning_handle_err() {
-        use std::cell::RefCell;
-        let cell = Rc::new(RefCell::new(2));
-        let cell_ref = RcRef::new(cell);
-        let handle = OwningHandle::try_new::<_, ()>(cell_ref, |x| {
-            if false {
-                return Ok(unsafe { x.as_ref() }.unwrap().borrow_mut());
-            }
-            Err(())
-        });
-        assert!(handle.is_err());
-    }
-
-    #[test]
-    fn nested() {
-        use std::cell::RefCell;
-        use std::sync::{Arc, RwLock};
-
-        let result = {
-            let complex = Rc::new(RefCell::new(Arc::new(RwLock::new("someString"))));
-            let curr = RcRef::new(complex);
-            let curr =
-                OwningHandle::new_with_fn(curr, |x| unsafe { x.as_ref() }.unwrap().borrow_mut());
-            let mut curr = OwningHandle::new_with_fn(curr, |x| {
-                unsafe { x.as_ref() }.unwrap().try_write().unwrap()
-            });
-            assert_eq!(*curr, "someString");
-            *curr = "someOtherString";
-            curr
-        };
-        assert_eq!(*result, "someOtherString");
-    }
-
-    #[test]
-    fn owning_handle_safe() {
-        use std::cell::RefCell;
-        let cell = Rc::new(RefCell::new(2));
-        let cell_ref = RcRef::new(cell);
-        let handle = OwningHandle::new(cell_ref);
-        assert_eq!(*handle, 2);
-    }
-
-    #[test]
-    fn owning_handle_mut_safe() {
-        use std::cell::RefCell;
-        let cell = Rc::new(RefCell::new(2));
-        let cell_ref = RcRef::new(cell);
-        let mut handle = OwningHandle::new_mut(cell_ref);
-        assert_eq!(*handle, 2);
-        *handle = 3;
-        assert_eq!(*handle, 3);
-    }
-
-    #[test]
-    fn owning_handle_safe_2() {
-        let result = {
-            let complex = Rc::new(RefCell::new(Arc::new(RwLock::new("someString"))));
-            let curr = RcRef::new(complex);
-            let curr =
-                OwningHandle::new_with_fn(curr, |x| unsafe { x.as_ref() }.unwrap().borrow_mut());
-            let mut curr = OwningHandle::new_with_fn(curr, |x| {
-                unsafe { x.as_ref() }.unwrap().try_write().unwrap()
-            });
-            assert_eq!(*curr, "someString");
-            *curr = "someOtherString";
-            curr
-        };
-        assert_eq!(*result, "someOtherString");
-    }
-}
-
-// FIXME: owning_ref is not sound under stacked borrows. Preferably, get rid of it.
-#[cfg(not(miri))]
-mod owning_ref_mut {
-    use super::super::BoxRef;
-    use super::super::{BoxRefMut, Erased, ErasedBoxRefMut, OwningRefMut};
-    use std::cmp::Ordering;
-    use std::collections::hash_map::DefaultHasher;
-    use std::collections::HashMap;
-    use std::hash::{Hash, Hasher};
-
-    #[derive(Debug, PartialEq)]
-    struct Example(u32, String, [u8; 3]);
-    fn example() -> Example {
-        Example(42, "hello world".to_string(), [1, 2, 3])
-    }
-
-    #[test]
-    fn new_deref() {
-        let or: OwningRefMut<Box<()>, ()> = OwningRefMut::new(Box::new(()));
-        assert_eq!(&*or, &());
-    }
-
-    #[test]
-    fn new_deref_mut() {
-        let mut or: OwningRefMut<Box<()>, ()> = OwningRefMut::new(Box::new(()));
-        assert_eq!(&mut *or, &mut ());
-    }
-
-    #[test]
-    fn mutate() {
-        let mut or: OwningRefMut<Box<usize>, usize> = OwningRefMut::new(Box::new(0));
-        assert_eq!(&*or, &0);
-        *or = 1;
-        assert_eq!(&*or, &1);
-    }
-
-    #[test]
-    fn into() {
-        let or: OwningRefMut<Box<()>, ()> = Box::new(()).into();
-        assert_eq!(&*or, &());
-    }
-
-    #[test]
-    fn map_offset_ref() {
-        let or: BoxRefMut<Example> = Box::new(example()).into();
-        let or: BoxRef<_, u32> = or.map(|x| &mut x.0);
-        assert_eq!(&*or, &42);
-
-        let or: BoxRefMut<Example> = Box::new(example()).into();
-        let or: BoxRef<_, u8> = or.map(|x| &mut x.2[1]);
-        assert_eq!(&*or, &2);
-    }
-
-    #[test]
-    fn map_heap_ref() {
-        let or: BoxRefMut<Example> = Box::new(example()).into();
-        let or: BoxRef<_, str> = or.map(|x| &mut x.1[..5]);
-        assert_eq!(&*or, "hello");
-    }
-
-    #[test]
-    fn map_static_ref() {
-        let or: BoxRefMut<()> = Box::new(()).into();
-        let or: BoxRef<_, str> = or.map(|_| "hello");
-        assert_eq!(&*or, "hello");
-    }
-
-    #[test]
-    fn map_mut_offset_ref() {
-        let or: BoxRefMut<Example> = Box::new(example()).into();
-        let or: BoxRefMut<_, u32> = or.map_mut(|x| &mut x.0);
-        assert_eq!(&*or, &42);
-
-        let or: BoxRefMut<Example> = Box::new(example()).into();
-        let or: BoxRefMut<_, u8> = or.map_mut(|x| &mut x.2[1]);
-        assert_eq!(&*or, &2);
-    }
-
-    #[test]
-    fn map_mut_heap_ref() {
-        let or: BoxRefMut<Example> = Box::new(example()).into();
-        let or: BoxRefMut<_, str> = or.map_mut(|x| &mut x.1[..5]);
-        assert_eq!(&*or, "hello");
-    }
-
-    #[test]
-    fn map_mut_static_ref() {
-        static mut MUT_S: [u8; 5] = *b"hello";
-
-        let mut_s: &'static mut [u8] = unsafe { &mut MUT_S };
-
-        let or: BoxRefMut<()> = Box::new(()).into();
-        let or: BoxRefMut<_, [u8]> = or.map_mut(move |_| mut_s);
-        assert_eq!(&*or, b"hello");
-    }
-
-    #[test]
-    fn map_mut_chained() {
-        let or: BoxRefMut<String> = Box::new(example().1).into();
-        let or: BoxRefMut<_, str> = or.map_mut(|x| &mut x[1..5]);
-        let or: BoxRefMut<_, str> = or.map_mut(|x| &mut x[..2]);
-        assert_eq!(&*or, "el");
-    }
-
-    #[test]
-    fn map_chained_inference() {
-        let or = BoxRefMut::new(Box::new(example().1))
-            .map_mut(|x| &mut x[..5])
-            .map_mut(|x| &mut x[1..3]);
-        assert_eq!(&*or, "el");
-    }
-
-    #[test]
-    fn try_map_mut() {
-        let or: BoxRefMut<String> = Box::new(example().1).into();
-        let or: Result<BoxRefMut<_, str>, ()> = or.try_map_mut(|x| Ok(&mut x[1..5]));
-        assert_eq!(&*or.unwrap(), "ello");
-
-        let or: BoxRefMut<String> = Box::new(example().1).into();
-        let or: Result<BoxRefMut<_, str>, ()> = or.try_map_mut(|_| Err(()));
-        assert!(or.is_err());
-    }
-
-    #[test]
-    fn owner() {
-        let or: BoxRefMut<String> = Box::new(example().1).into();
-        let or = or.map_mut(|x| &mut x[..5]);
-        assert_eq!(&*or, "hello");
-        assert_eq!(&**or.owner(), "hello world");
-    }
-
-    #[test]
-    fn into_inner() {
-        let or: BoxRefMut<String> = Box::new(example().1).into();
-        let or = or.map_mut(|x| &mut x[..5]);
-        assert_eq!(&*or, "hello");
-        let s = *or.into_inner();
-        assert_eq!(&s, "hello world");
-    }
-
-    #[test]
-    fn fmt_debug() {
-        let or: BoxRefMut<String> = Box::new(example().1).into();
-        let or = or.map_mut(|x| &mut x[..5]);
-        let s = format!("{:?}", or);
-        assert_eq!(&s, "OwningRefMut { owner: \"hello world\", reference: \"hello\" }");
-    }
-
-    #[test]
-    fn erased_owner() {
-        let o1: BoxRefMut<Example, str> =
-            BoxRefMut::new(Box::new(example())).map_mut(|x| &mut x.1[..]);
-
-        let o2: BoxRefMut<String, str> =
-            BoxRefMut::new(Box::new(example().1)).map_mut(|x| &mut x[..]);
-
-        let os: Vec<ErasedBoxRefMut<str>> = vec![o1.erase_owner(), o2.erase_owner()];
-        assert!(os.iter().all(|e| &e[..] == "hello world"));
-    }
-
-    #[test]
-    fn raii_locks() {
-        use super::super::RefMutRefMut;
-        use super::super::{MutexGuardRefMut, RwLockWriteGuardRefMut};
-        use std::cell::RefCell;
-        use std::sync::{Mutex, RwLock};
-
-        {
-            let a = RefCell::new(1);
-            let a = {
-                let a = RefMutRefMut::new(a.borrow_mut());
-                assert_eq!(*a, 1);
-                a
-            };
-            assert_eq!(*a, 1);
-            drop(a);
-        }
-        {
-            let a = Mutex::new(1);
-            let a = {
-                let a = MutexGuardRefMut::new(a.lock().unwrap());
-                assert_eq!(*a, 1);
-                a
-            };
-            assert_eq!(*a, 1);
-            drop(a);
-        }
-        {
-            let a = RwLock::new(1);
-            let a = {
-                let a = RwLockWriteGuardRefMut::new(a.write().unwrap());
-                assert_eq!(*a, 1);
-                a
-            };
-            assert_eq!(*a, 1);
-            drop(a);
-        }
-    }
-
-    #[test]
-    fn eq() {
-        let or1: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice());
-        let or2: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice());
-        assert_eq!(or1.eq(&or2), true);
-    }
-
-    #[test]
-    fn cmp() {
-        let or1: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice());
-        let or2: BoxRefMut<[u8]> = BoxRefMut::new(vec![4, 5, 6].into_boxed_slice());
-        assert_eq!(or1.cmp(&or2), Ordering::Less);
-    }
-
-    #[test]
-    fn partial_cmp() {
-        let or1: BoxRefMut<[u8]> = BoxRefMut::new(vec![4, 5, 6].into_boxed_slice());
-        let or2: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice());
-        assert_eq!(or1.partial_cmp(&or2), Some(Ordering::Greater));
-    }
-
-    #[test]
-    fn hash() {
-        let mut h1 = DefaultHasher::new();
-        let mut h2 = DefaultHasher::new();
-
-        let or1: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice());
-        let or2: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice());
-
-        or1.hash(&mut h1);
-        or2.hash(&mut h2);
-
-        assert_eq!(h1.finish(), h2.finish());
-    }
-
-    #[test]
-    fn borrow() {
-        let mut hash = HashMap::new();
-        let key1 = BoxRefMut::<String>::new(Box::new("foo".to_string())).map(|s| &s[..]);
-        let key2 = BoxRefMut::<String>::new(Box::new("bar".to_string())).map(|s| &s[..]);
-
-        hash.insert(key1, 42);
-        hash.insert(key2, 23);
-
-        assert_eq!(hash.get("foo"), Some(&42));
-        assert_eq!(hash.get("bar"), Some(&23));
-    }
-
-    #[test]
-    fn total_erase() {
-        let a: OwningRefMut<Vec<u8>, [u8]> = OwningRefMut::new(vec![]).map_mut(|x| &mut x[..]);
-        let b: OwningRefMut<Box<[u8]>, [u8]> =
-            OwningRefMut::new(vec![].into_boxed_slice()).map_mut(|x| &mut x[..]);
-
-        let c: OwningRefMut<Box<Vec<u8>>, [u8]> = unsafe { a.map_owner(Box::new) };
-        let d: OwningRefMut<Box<Box<[u8]>>, [u8]> = unsafe { b.map_owner(Box::new) };
-
-        let _e: OwningRefMut<Box<dyn Erased>, [u8]> = c.erase_owner();
-        let _f: OwningRefMut<Box<dyn Erased>, [u8]> = d.erase_owner();
-    }
-
-    #[test]
-    fn total_erase_box() {
-        let a: OwningRefMut<Vec<u8>, [u8]> = OwningRefMut::new(vec![]).map_mut(|x| &mut x[..]);
-        let b: OwningRefMut<Box<[u8]>, [u8]> =
-            OwningRefMut::new(vec![].into_boxed_slice()).map_mut(|x| &mut x[..]);
-
-        let c: OwningRefMut<Box<Vec<u8>>, [u8]> = a.map_owner_box();
-        let d: OwningRefMut<Box<Box<[u8]>>, [u8]> = b.map_owner_box();
-
-        let _e: OwningRefMut<Box<dyn Erased>, [u8]> = c.erase_owner();
-        let _f: OwningRefMut<Box<dyn Erased>, [u8]> = d.erase_owner();
-    }
-
-    #[test]
-    fn try_map1() {
-        use std::any::Any;
-
-        let x = Box::new(123_i32);
-        let y: Box<dyn Any> = x;
-
-        assert!(OwningRefMut::new(y).try_map_mut(|x| x.downcast_mut::<i32>().ok_or(())).is_ok());
-    }
-
-    #[test]
-    fn try_map2() {
-        use std::any::Any;
-
-        let x = Box::new(123_i32);
-        let y: Box<dyn Any> = x;
-
-        assert!(!OwningRefMut::new(y).try_map_mut(|x| x.downcast_mut::<i32>().ok_or(())).is_err());
-    }
-
-    #[test]
-    fn try_map3() {
-        use std::any::Any;
-
-        let x = Box::new(123_i32);
-        let y: Box<dyn Any> = x;
-
-        assert!(OwningRefMut::new(y).try_map(|x| x.downcast_ref::<i32>().ok_or(())).is_ok());
-    }
-
-    #[test]
-    fn try_map4() {
-        use std::any::Any;
-
-        let x = Box::new(123_i32);
-        let y: Box<dyn Any> = x;
-
-        assert!(!OwningRefMut::new(y).try_map(|x| x.downcast_ref::<i32>().ok_or(())).is_err());
-    }
-
-    #[test]
-    fn into_owning_ref() {
-        use super::super::BoxRef;
-
-        let or: BoxRefMut<()> = Box::new(()).into();
-        let or: BoxRef<()> = or.into();
-        assert_eq!(&*or, &());
-    }
-
-    struct Foo {
-        u: u32,
-    }
-    struct Bar {
-        f: Foo,
-    }
-
-    #[test]
-    fn ref_mut() {
-        use std::cell::RefCell;
-
-        let a = RefCell::new(Bar { f: Foo { u: 42 } });
-        let mut b = OwningRefMut::new(a.borrow_mut());
-        assert_eq!(b.f.u, 42);
-        b.f.u = 43;
-        let mut c = b.map_mut(|x| &mut x.f);
-        assert_eq!(c.u, 43);
-        c.u = 44;
-        let mut d = c.map_mut(|x| &mut x.u);
-        assert_eq!(*d, 44);
-        *d = 45;
-        assert_eq!(*d, 45);
-    }
-}
diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs
index 443316836..1ed584eaf 100644
--- a/compiler/rustc_data_structures/src/profiling.rs
+++ b/compiler/rustc_data_structures/src/profiling.rs
@@ -97,6 +97,7 @@ use std::time::{Duration, Instant};
 pub use measureme::EventId;
 use measureme::{EventIdBuilder, Profiler, SerializableString, StringId};
 use parking_lot::RwLock;
+use serde_json::json;
 use smallvec::SmallVec;
 
 bitflags::bitflags! {
@@ -145,6 +146,15 @@ const EVENT_FILTERS_BY_NAME: &[(&str, EventFilter)] = &[
 /// Something that uniquely identifies a query invocation.
 pub struct QueryInvocationId(pub u32);
 
+/// Which format to use for `-Z time-passes`
+#[derive(Clone, Copy, PartialEq, Hash, Debug)]
+pub enum TimePassesFormat {
+    /// Emit human readable text
+    Text,
+    /// Emit structured JSON
+    Json,
+}
+
 /// A reference to the SelfProfiler. It can be cloned and sent across thread
 /// boundaries at will.
 #[derive(Clone)]
@@ -158,14 +168,14 @@ pub struct SelfProfilerRef {
     // actually enabled.
     event_filter_mask: EventFilter,
 
-    // Print verbose generic activities to stderr?
-    print_verbose_generic_activities: bool,
+    // Print verbose generic activities to stderr.
+    print_verbose_generic_activities: Option<TimePassesFormat>,
 }
 
 impl SelfProfilerRef {
     pub fn new(
         profiler: Option<Arc<SelfProfiler>>,
-        print_verbose_generic_activities: bool,
+        print_verbose_generic_activities: Option<TimePassesFormat>,
     ) -> SelfProfilerRef {
         // If there is no SelfProfiler then the filter mask is set to NONE,
         // ensuring that nothing ever tries to actually access it.
@@ -207,9 +217,10 @@ impl SelfProfilerRef {
     /// a measureme event, "verbose" generic activities also print a timing entry to
     /// stderr if the compiler is invoked with -Ztime-passes.
     pub fn verbose_generic_activity(&self, event_label: &'static str) -> VerboseTimingGuard<'_> {
-        let message = self.print_verbose_generic_activities.then(|| event_label.to_owned());
+        let message_and_format =
+            self.print_verbose_generic_activities.map(|format| (event_label.to_owned(), format));
 
-        VerboseTimingGuard::start(message, self.generic_activity(event_label))
+        VerboseTimingGuard::start(message_and_format, self.generic_activity(event_label))
     }
 
     /// Like `verbose_generic_activity`, but with an extra arg.
@@ -221,11 +232,14 @@ impl SelfProfilerRef {
     where
         A: Borrow<str> + Into<String>,
     {
-        let message = self
+        let message_and_format = self
             .print_verbose_generic_activities
-            .then(|| format!("{}({})", event_label, event_arg.borrow()));
+            .map(|format| (format!("{}({})", event_label, event_arg.borrow()), format));
 
-        VerboseTimingGuard::start(message, self.generic_activity_with_arg(event_label, event_arg))
+        VerboseTimingGuard::start(
+            message_and_format,
+            self.generic_activity_with_arg(event_label, event_arg),
+        )
     }
 
     /// Start profiling a generic activity. Profiling continues until the
@@ -703,17 +717,32 @@ impl<'a> TimingGuard<'a> {
     }
 }
 
+struct VerboseInfo {
+    start_time: Instant,
+    start_rss: Option<usize>,
+    message: String,
+    format: TimePassesFormat,
+}
+
 #[must_use]
 pub struct VerboseTimingGuard<'a> {
-    start_and_message: Option<(Instant, Option<usize>, String)>,
+    info: Option<VerboseInfo>,
     _guard: TimingGuard<'a>,
 }
 
 impl<'a> VerboseTimingGuard<'a> {
-    pub fn start(message: Option<String>, _guard: TimingGuard<'a>) -> Self {
+    pub fn start(
+        message_and_format: Option<(String, TimePassesFormat)>,
+        _guard: TimingGuard<'a>,
+    ) -> Self {
         VerboseTimingGuard {
             _guard,
-            start_and_message: message.map(|msg| (Instant::now(), get_resident_set_size(), msg)),
+            info: message_and_format.map(|(message, format)| VerboseInfo {
+                start_time: Instant::now(),
+                start_rss: get_resident_set_size(),
+                message,
+                format,
+            }),
         }
     }
 
@@ -726,10 +755,10 @@ impl<'a> VerboseTimingGuard<'a> {
 
 impl Drop for VerboseTimingGuard<'_> {
     fn drop(&mut self) {
-        if let Some((start_time, start_rss, ref message)) = self.start_and_message {
+        if let Some(info) = &self.info {
             let end_rss = get_resident_set_size();
-            let dur = start_time.elapsed();
-            print_time_passes_entry(message, dur, start_rss, end_rss);
+            let dur = info.start_time.elapsed();
+            print_time_passes_entry(&info.message, dur, info.start_rss, end_rss, info.format);
         }
     }
 }
@@ -739,7 +768,22 @@ pub fn print_time_passes_entry(
     dur: Duration,
     start_rss: Option<usize>,
     end_rss: Option<usize>,
+    format: TimePassesFormat,
 ) {
+    match format {
+        TimePassesFormat::Json => {
+            let json = json!({
+                "pass": what,
+                "time": dur.as_secs_f64(),
+                "rss_start": start_rss,
+                "rss_end": end_rss,
+            });
+            eprintln!("time: {json}");
+            return;
+        }
+        TimePassesFormat::Text => (),
+    }
+
     // Print the pass if its duration is greater than 5 ms, or it changed the
     // measured RSS.
     let is_notable = || {
@@ -796,21 +840,26 @@ fn get_thread_id() -> u32 {
 cfg_if! {
     if #[cfg(windows)] {
         pub fn get_resident_set_size() -> Option<usize> {
-            use std::mem::{self, MaybeUninit};
-            use winapi::shared::minwindef::DWORD;
-            use winapi::um::processthreadsapi::GetCurrentProcess;
-            use winapi::um::psapi::{GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS};
-
-            let mut pmc = MaybeUninit::<PROCESS_MEMORY_COUNTERS>::uninit();
-            match unsafe {
-                GetProcessMemoryInfo(GetCurrentProcess(), pmc.as_mut_ptr(), mem::size_of_val(&pmc) as DWORD)
-            } {
-                0 => None,
-                _ => {
-                    let pmc = unsafe { pmc.assume_init() };
-                    Some(pmc.WorkingSetSize as usize)
-                }
+            use std::mem;
+
+            use windows::{
+                Win32::System::ProcessStatus::{K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS},
+                Win32::System::Threading::GetCurrentProcess,
+            };
+
+            let mut pmc = PROCESS_MEMORY_COUNTERS::default();
+            let pmc_size = mem::size_of_val(&pmc);
+            unsafe {
+                K32GetProcessMemoryInfo(
+                    GetCurrentProcess(),
+                    &mut pmc,
+                    pmc_size as u32,
+                )
             }
+            .ok()
+            .ok()?;
+
+            Some(pmc.WorkingSetSize)
         }
     } else if #[cfg(target_os = "macos")] {
         pub fn get_resident_set_size() -> Option<usize> {
diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs
index 01d292dde..bd7a86f67 100644
--- a/compiler/rustc_data_structures/src/sharded.rs
+++ b/compiler/rustc_data_structures/src/sharded.rs
@@ -5,7 +5,7 @@ use std::collections::hash_map::RawEntryMut;
 use std::hash::{Hash, Hasher};
 use std::mem;
 
-#[derive(Clone, Default)]
+#[derive(Default)]
 #[cfg_attr(parallel_compiler, repr(align(64)))]
 struct CacheAligned<T>(T);
 
@@ -21,7 +21,6 @@ const SHARD_BITS: usize = 0;
 pub const SHARDS: usize = 1 << SHARD_BITS;
 
 /// An array of cache-line aligned inner locked structures with convenience methods.
-#[derive(Clone)]
 pub struct Sharded<T> {
     shards: [CacheAligned<Lock<T>>; SHARDS],
 }
@@ -141,6 +140,7 @@ pub fn make_hash<K: Hash + ?Sized>(val: &K) -> u64 {
 /// `hash` can be computed with any hasher, so long as that hasher is used
 /// consistently for each `Sharded` instance.
 #[inline]
+#[allow(clippy::modulo_one)]
 pub fn get_shard_index_by_hash(hash: u64) -> usize {
     let hash_len = mem::size_of::<usize>();
     // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits.
diff --git a/compiler/rustc_data_structures/src/sip128.rs b/compiler/rustc_data_structures/src/sip128.rs
index 90793a97e..d849fe037 100644
--- a/compiler/rustc_data_structures/src/sip128.rs
+++ b/compiler/rustc_data_structures/src/sip128.rs
@@ -247,7 +247,7 @@ impl SipHasher128 {
         for i in 0..BUFFER_CAPACITY {
             let elem = self.buf.get_unchecked(i).assume_init().to_le();
             self.state.v3 ^= elem;
-            Sip24Rounds::c_rounds(&mut self.state);
+            Sip13Rounds::c_rounds(&mut self.state);
             self.state.v0 ^= elem;
         }
 
@@ -327,7 +327,7 @@ impl SipHasher128 {
         for i in 0..last {
             let elem = self.buf.get_unchecked(i).assume_init().to_le();
             self.state.v3 ^= elem;
-            Sip24Rounds::c_rounds(&mut self.state);
+            Sip13Rounds::c_rounds(&mut self.state);
             self.state.v0 ^= elem;
         }
 
@@ -340,7 +340,7 @@ impl SipHasher128 {
         for _ in 0..elems_left {
             let elem = (msg.as_ptr().add(processed) as *const u64).read_unaligned().to_le();
             self.state.v3 ^= elem;
-            Sip24Rounds::c_rounds(&mut self.state);
+            Sip13Rounds::c_rounds(&mut self.state);
             self.state.v0 ^= elem;
             processed += ELEM_SIZE;
         }
@@ -368,7 +368,7 @@ impl SipHasher128 {
         for i in 0..last {
             let elem = unsafe { self.buf.get_unchecked(i).assume_init().to_le() };
             state.v3 ^= elem;
-            Sip24Rounds::c_rounds(&mut state);
+            Sip13Rounds::c_rounds(&mut state);
             state.v0 ^= elem;
         }
 
@@ -392,15 +392,15 @@ impl SipHasher128 {
         let b: u64 = ((length as u64 & 0xff) << 56) | elem;
 
         state.v3 ^= b;
-        Sip24Rounds::c_rounds(&mut state);
+        Sip13Rounds::c_rounds(&mut state);
         state.v0 ^= b;
 
         state.v2 ^= 0xee;
-        Sip24Rounds::d_rounds(&mut state);
+        Sip13Rounds::d_rounds(&mut state);
         let _0 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3;
 
         state.v1 ^= 0xdd;
-        Sip24Rounds::d_rounds(&mut state);
+        Sip13Rounds::d_rounds(&mut state);
         let _1 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3;
 
         (_0, _1)
@@ -477,13 +477,12 @@ impl Hasher for SipHasher128 {
 }
 
 #[derive(Debug, Clone, Default)]
-struct Sip24Rounds;
+struct Sip13Rounds;
 
-impl Sip24Rounds {
+impl Sip13Rounds {
     #[inline]
     fn c_rounds(state: &mut State) {
         compress!(state);
-        compress!(state);
     }
 
     #[inline]
@@ -491,6 +490,5 @@ impl Sip24Rounds {
         compress!(state);
         compress!(state);
         compress!(state);
-        compress!(state);
     }
 }
diff --git a/compiler/rustc_data_structures/src/sip128/tests.rs b/compiler/rustc_data_structures/src/sip128/tests.rs
index 5fe967c41..cc6d3b0f4 100644
--- a/compiler/rustc_data_structures/src/sip128/tests.rs
+++ b/compiler/rustc_data_structures/src/sip128/tests.rs
@@ -22,269 +22,76 @@ fn hash_with<T: Hash>(mut st: SipHasher128, x: &T) -> (u64, u64) {
 fn hash<T: Hash>(x: &T) -> (u64, u64) {
     hash_with(SipHasher128::new_with_keys(0, 0), x)
 }
-
+#[rustfmt::skip]
 const TEST_VECTOR: [[u8; 16]; 64] = [
-    [
-        0xa3, 0x81, 0x7f, 0x04, 0xba, 0x25, 0xa8, 0xe6, 0x6d, 0xf6, 0x72, 0x14, 0xc7, 0x55, 0x02,
-        0x93,
-    ],
-    [
-        0xda, 0x87, 0xc1, 0xd8, 0x6b, 0x99, 0xaf, 0x44, 0x34, 0x76, 0x59, 0x11, 0x9b, 0x22, 0xfc,
-        0x45,
-    ],
-    [
-        0x81, 0x77, 0x22, 0x8d, 0xa4, 0xa4, 0x5d, 0xc7, 0xfc, 0xa3, 0x8b, 0xde, 0xf6, 0x0a, 0xff,
-        0xe4,
-    ],
-    [
-        0x9c, 0x70, 0xb6, 0x0c, 0x52, 0x67, 0xa9, 0x4e, 0x5f, 0x33, 0xb6, 0xb0, 0x29, 0x85, 0xed,
-        0x51,
-    ],
-    [
-        0xf8, 0x81, 0x64, 0xc1, 0x2d, 0x9c, 0x8f, 0xaf, 0x7d, 0x0f, 0x6e, 0x7c, 0x7b, 0xcd, 0x55,
-        0x79,
-    ],
-    [
-        0x13, 0x68, 0x87, 0x59, 0x80, 0x77, 0x6f, 0x88, 0x54, 0x52, 0x7a, 0x07, 0x69, 0x0e, 0x96,
-        0x27,
-    ],
-    [
-        0x14, 0xee, 0xca, 0x33, 0x8b, 0x20, 0x86, 0x13, 0x48, 0x5e, 0xa0, 0x30, 0x8f, 0xd7, 0xa1,
-        0x5e,
-    ],
-    [
-        0xa1, 0xf1, 0xeb, 0xbe, 0xd8, 0xdb, 0xc1, 0x53, 0xc0, 0xb8, 0x4a, 0xa6, 0x1f, 0xf0, 0x82,
-        0x39,
-    ],
-    [
-        0x3b, 0x62, 0xa9, 0xba, 0x62, 0x58, 0xf5, 0x61, 0x0f, 0x83, 0xe2, 0x64, 0xf3, 0x14, 0x97,
-        0xb4,
-    ],
-    [
-        0x26, 0x44, 0x99, 0x06, 0x0a, 0xd9, 0xba, 0xab, 0xc4, 0x7f, 0x8b, 0x02, 0xbb, 0x6d, 0x71,
-        0xed,
-    ],
-    [
-        0x00, 0x11, 0x0d, 0xc3, 0x78, 0x14, 0x69, 0x56, 0xc9, 0x54, 0x47, 0xd3, 0xf3, 0xd0, 0xfb,
-        0xba,
-    ],
-    [
-        0x01, 0x51, 0xc5, 0x68, 0x38, 0x6b, 0x66, 0x77, 0xa2, 0xb4, 0xdc, 0x6f, 0x81, 0xe5, 0xdc,
-        0x18,
-    ],
-    [
-        0xd6, 0x26, 0xb2, 0x66, 0x90, 0x5e, 0xf3, 0x58, 0x82, 0x63, 0x4d, 0xf6, 0x85, 0x32, 0xc1,
-        0x25,
-    ],
-    [
-        0x98, 0x69, 0xe2, 0x47, 0xe9, 0xc0, 0x8b, 0x10, 0xd0, 0x29, 0x93, 0x4f, 0xc4, 0xb9, 0x52,
-        0xf7,
-    ],
-    [
-        0x31, 0xfc, 0xef, 0xac, 0x66, 0xd7, 0xde, 0x9c, 0x7e, 0xc7, 0x48, 0x5f, 0xe4, 0x49, 0x49,
-        0x02,
-    ],
-    [
-        0x54, 0x93, 0xe9, 0x99, 0x33, 0xb0, 0xa8, 0x11, 0x7e, 0x08, 0xec, 0x0f, 0x97, 0xcf, 0xc3,
-        0xd9,
-    ],
-    [
-        0x6e, 0xe2, 0xa4, 0xca, 0x67, 0xb0, 0x54, 0xbb, 0xfd, 0x33, 0x15, 0xbf, 0x85, 0x23, 0x05,
-        0x77,
-    ],
-    [
-        0x47, 0x3d, 0x06, 0xe8, 0x73, 0x8d, 0xb8, 0x98, 0x54, 0xc0, 0x66, 0xc4, 0x7a, 0xe4, 0x77,
-        0x40,
-    ],
-    [
-        0xa4, 0x26, 0xe5, 0xe4, 0x23, 0xbf, 0x48, 0x85, 0x29, 0x4d, 0xa4, 0x81, 0xfe, 0xae, 0xf7,
-        0x23,
-    ],
-    [
-        0x78, 0x01, 0x77, 0x31, 0xcf, 0x65, 0xfa, 0xb0, 0x74, 0xd5, 0x20, 0x89, 0x52, 0x51, 0x2e,
-        0xb1,
-    ],
-    [
-        0x9e, 0x25, 0xfc, 0x83, 0x3f, 0x22, 0x90, 0x73, 0x3e, 0x93, 0x44, 0xa5, 0xe8, 0x38, 0x39,
-        0xeb,
-    ],
-    [
-        0x56, 0x8e, 0x49, 0x5a, 0xbe, 0x52, 0x5a, 0x21, 0x8a, 0x22, 0x14, 0xcd, 0x3e, 0x07, 0x1d,
-        0x12,
-    ],
-    [
-        0x4a, 0x29, 0xb5, 0x45, 0x52, 0xd1, 0x6b, 0x9a, 0x46, 0x9c, 0x10, 0x52, 0x8e, 0xff, 0x0a,
-        0xae,
-    ],
-    [
-        0xc9, 0xd1, 0x84, 0xdd, 0xd5, 0xa9, 0xf5, 0xe0, 0xcf, 0x8c, 0xe2, 0x9a, 0x9a, 0xbf, 0x69,
-        0x1c,
-    ],
-    [
-        0x2d, 0xb4, 0x79, 0xae, 0x78, 0xbd, 0x50, 0xd8, 0x88, 0x2a, 0x8a, 0x17, 0x8a, 0x61, 0x32,
-        0xad,
-    ],
-    [
-        0x8e, 0xce, 0x5f, 0x04, 0x2d, 0x5e, 0x44, 0x7b, 0x50, 0x51, 0xb9, 0xea, 0xcb, 0x8d, 0x8f,
-        0x6f,
-    ],
-    [
-        0x9c, 0x0b, 0x53, 0xb4, 0xb3, 0xc3, 0x07, 0xe8, 0x7e, 0xae, 0xe0, 0x86, 0x78, 0x14, 0x1f,
-        0x66,
-    ],
-    [
-        0xab, 0xf2, 0x48, 0xaf, 0x69, 0xa6, 0xea, 0xe4, 0xbf, 0xd3, 0xeb, 0x2f, 0x12, 0x9e, 0xeb,
-        0x94,
-    ],
-    [
-        0x06, 0x64, 0xda, 0x16, 0x68, 0x57, 0x4b, 0x88, 0xb9, 0x35, 0xf3, 0x02, 0x73, 0x58, 0xae,
-        0xf4,
-    ],
-    [
-        0xaa, 0x4b, 0x9d, 0xc4, 0xbf, 0x33, 0x7d, 0xe9, 0x0c, 0xd4, 0xfd, 0x3c, 0x46, 0x7c, 0x6a,
-        0xb7,
-    ],
-    [
-        0xea, 0x5c, 0x7f, 0x47, 0x1f, 0xaf, 0x6b, 0xde, 0x2b, 0x1a, 0xd7, 0xd4, 0x68, 0x6d, 0x22,
-        0x87,
-    ],
-    [
-        0x29, 0x39, 0xb0, 0x18, 0x32, 0x23, 0xfa, 0xfc, 0x17, 0x23, 0xde, 0x4f, 0x52, 0xc4, 0x3d,
-        0x35,
-    ],
-    [
-        0x7c, 0x39, 0x56, 0xca, 0x5e, 0xea, 0xfc, 0x3e, 0x36, 0x3e, 0x9d, 0x55, 0x65, 0x46, 0xeb,
-        0x68,
-    ],
-    [
-        0x77, 0xc6, 0x07, 0x71, 0x46, 0xf0, 0x1c, 0x32, 0xb6, 0xb6, 0x9d, 0x5f, 0x4e, 0xa9, 0xff,
-        0xcf,
-    ],
-    [
-        0x37, 0xa6, 0x98, 0x6c, 0xb8, 0x84, 0x7e, 0xdf, 0x09, 0x25, 0xf0, 0xf1, 0x30, 0x9b, 0x54,
-        0xde,
-    ],
-    [
-        0xa7, 0x05, 0xf0, 0xe6, 0x9d, 0xa9, 0xa8, 0xf9, 0x07, 0x24, 0x1a, 0x2e, 0x92, 0x3c, 0x8c,
-        0xc8,
-    ],
-    [
-        0x3d, 0xc4, 0x7d, 0x1f, 0x29, 0xc4, 0x48, 0x46, 0x1e, 0x9e, 0x76, 0xed, 0x90, 0x4f, 0x67,
-        0x11,
-    ],
-    [
-        0x0d, 0x62, 0xbf, 0x01, 0xe6, 0xfc, 0x0e, 0x1a, 0x0d, 0x3c, 0x47, 0x51, 0xc5, 0xd3, 0x69,
-        0x2b,
-    ],
-    [
-        0x8c, 0x03, 0x46, 0x8b, 0xca, 0x7c, 0x66, 0x9e, 0xe4, 0xfd, 0x5e, 0x08, 0x4b, 0xbe, 0xe7,
-        0xb5,
-    ],
-    [
-        0x52, 0x8a, 0x5b, 0xb9, 0x3b, 0xaf, 0x2c, 0x9c, 0x44, 0x73, 0xcc, 0xe5, 0xd0, 0xd2, 0x2b,
-        0xd9,
-    ],
-    [
-        0xdf, 0x6a, 0x30, 0x1e, 0x95, 0xc9, 0x5d, 0xad, 0x97, 0xae, 0x0c, 0xc8, 0xc6, 0x91, 0x3b,
-        0xd8,
-    ],
-    [
-        0x80, 0x11, 0x89, 0x90, 0x2c, 0x85, 0x7f, 0x39, 0xe7, 0x35, 0x91, 0x28, 0x5e, 0x70, 0xb6,
-        0xdb,
-    ],
-    [
-        0xe6, 0x17, 0x34, 0x6a, 0xc9, 0xc2, 0x31, 0xbb, 0x36, 0x50, 0xae, 0x34, 0xcc, 0xca, 0x0c,
-        0x5b,
-    ],
-    [
-        0x27, 0xd9, 0x34, 0x37, 0xef, 0xb7, 0x21, 0xaa, 0x40, 0x18, 0x21, 0xdc, 0xec, 0x5a, 0xdf,
-        0x89,
-    ],
-    [
-        0x89, 0x23, 0x7d, 0x9d, 0xed, 0x9c, 0x5e, 0x78, 0xd8, 0xb1, 0xc9, 0xb1, 0x66, 0xcc, 0x73,
-        0x42,
-    ],
-    [
-        0x4a, 0x6d, 0x80, 0x91, 0xbf, 0x5e, 0x7d, 0x65, 0x11, 0x89, 0xfa, 0x94, 0xa2, 0x50, 0xb1,
-        0x4c,
-    ],
-    [
-        0x0e, 0x33, 0xf9, 0x60, 0x55, 0xe7, 0xae, 0x89, 0x3f, 0xfc, 0x0e, 0x3d, 0xcf, 0x49, 0x29,
-        0x02,
-    ],
-    [
-        0xe6, 0x1c, 0x43, 0x2b, 0x72, 0x0b, 0x19, 0xd1, 0x8e, 0xc8, 0xd8, 0x4b, 0xdc, 0x63, 0x15,
-        0x1b,
-    ],
-    [
-        0xf7, 0xe5, 0xae, 0xf5, 0x49, 0xf7, 0x82, 0xcf, 0x37, 0x90, 0x55, 0xa6, 0x08, 0x26, 0x9b,
-        0x16,
-    ],
-    [
-        0x43, 0x8d, 0x03, 0x0f, 0xd0, 0xb7, 0xa5, 0x4f, 0xa8, 0x37, 0xf2, 0xad, 0x20, 0x1a, 0x64,
-        0x03,
-    ],
-    [
-        0xa5, 0x90, 0xd3, 0xee, 0x4f, 0xbf, 0x04, 0xe3, 0x24, 0x7e, 0x0d, 0x27, 0xf2, 0x86, 0x42,
-        0x3f,
-    ],
-    [
-        0x5f, 0xe2, 0xc1, 0xa1, 0x72, 0xfe, 0x93, 0xc4, 0xb1, 0x5c, 0xd3, 0x7c, 0xae, 0xf9, 0xf5,
-        0x38,
-    ],
-    [
-        0x2c, 0x97, 0x32, 0x5c, 0xbd, 0x06, 0xb3, 0x6e, 0xb2, 0x13, 0x3d, 0xd0, 0x8b, 0x3a, 0x01,
-        0x7c,
-    ],
-    [
-        0x92, 0xc8, 0x14, 0x22, 0x7a, 0x6b, 0xca, 0x94, 0x9f, 0xf0, 0x65, 0x9f, 0x00, 0x2a, 0xd3,
-        0x9e,
-    ],
-    [
-        0xdc, 0xe8, 0x50, 0x11, 0x0b, 0xd8, 0x32, 0x8c, 0xfb, 0xd5, 0x08, 0x41, 0xd6, 0x91, 0x1d,
-        0x87,
-    ],
-    [
-        0x67, 0xf1, 0x49, 0x84, 0xc7, 0xda, 0x79, 0x12, 0x48, 0xe3, 0x2b, 0xb5, 0x92, 0x25, 0x83,
-        0xda,
-    ],
-    [
-        0x19, 0x38, 0xf2, 0xcf, 0x72, 0xd5, 0x4e, 0xe9, 0x7e, 0x94, 0x16, 0x6f, 0xa9, 0x1d, 0x2a,
-        0x36,
-    ],
-    [
-        0x74, 0x48, 0x1e, 0x96, 0x46, 0xed, 0x49, 0xfe, 0x0f, 0x62, 0x24, 0x30, 0x16, 0x04, 0x69,
-        0x8e,
-    ],
-    [
-        0x57, 0xfc, 0xa5, 0xde, 0x98, 0xa9, 0xd6, 0xd8, 0x00, 0x64, 0x38, 0xd0, 0x58, 0x3d, 0x8a,
-        0x1d,
-    ],
-    [
-        0x9f, 0xec, 0xde, 0x1c, 0xef, 0xdc, 0x1c, 0xbe, 0xd4, 0x76, 0x36, 0x74, 0xd9, 0x57, 0x53,
-        0x59,
-    ],
-    [
-        0xe3, 0x04, 0x0c, 0x00, 0xeb, 0x28, 0xf1, 0x53, 0x66, 0xca, 0x73, 0xcb, 0xd8, 0x72, 0xe7,
-        0x40,
-    ],
-    [
-        0x76, 0x97, 0x00, 0x9a, 0x6a, 0x83, 0x1d, 0xfe, 0xcc, 0xa9, 0x1c, 0x59, 0x93, 0x67, 0x0f,
-        0x7a,
-    ],
-    [
-        0x58, 0x53, 0x54, 0x23, 0x21, 0xf5, 0x67, 0xa0, 0x05, 0xd5, 0x47, 0xa4, 0xf0, 0x47, 0x59,
-        0xbd,
-    ],
-    [
-        0x51, 0x50, 0xd1, 0x77, 0x2f, 0x50, 0x83, 0x4a, 0x50, 0x3e, 0x06, 0x9a, 0x97, 0x3f, 0xbd,
-        0x7c,
-    ],
+    [0xe7, 0x7e, 0xbc, 0xb2, 0x27, 0x88, 0xa5, 0xbe, 0xfd, 0x62, 0xdb, 0x6a, 0xdd, 0x30, 0x30, 0x01],
+    [0xfc, 0x6f, 0x37, 0x04, 0x60, 0xd3, 0xed, 0xa8, 0x5e, 0x05, 0x73, 0xcc, 0x2b, 0x2f, 0xf0, 0x63],
+    [0x75, 0x78, 0x7f, 0x09, 0x05, 0x69, 0x83, 0x9b, 0x85, 0x5b, 0xc9, 0x54, 0x8c, 0x6a, 0xea, 0x95],
+    [0x6b, 0xc5, 0xcc, 0xfa, 0x1e, 0xdc, 0xf7, 0x9f, 0x48, 0x23, 0x18, 0x77, 0x12, 0xeb, 0xd7, 0x43],
+    [0x0c, 0x78, 0x4e, 0x71, 0xac, 0x2b, 0x28, 0x5a, 0x9f, 0x8e, 0x92, 0xe7, 0x8f, 0xbf, 0x2c, 0x25],
+    [0xf3, 0x28, 0xdb, 0x89, 0x34, 0x5b, 0x62, 0x0c, 0x79, 0x52, 0x29, 0xa4, 0x26, 0x95, 0x84, 0x3e],
+    [0xdc, 0xd0, 0x3d, 0x29, 0xf7, 0x43, 0xe7, 0x10, 0x09, 0x51, 0xb0, 0xe8, 0x39, 0x85, 0xa6, 0xf8],
+    [0x10, 0x84, 0xb9, 0x23, 0xf2, 0xaa, 0xe0, 0xc3, 0xa6, 0x2f, 0x2e, 0xc8, 0x08, 0x48, 0xab, 0x77],
+    [0xaa, 0x12, 0xfe, 0xe1, 0xd5, 0xe3, 0xda, 0xb4, 0x72, 0x4f, 0x16, 0xab, 0x35, 0xf9, 0xc7, 0x99],
+    [0x81, 0xdd, 0xb8, 0x04, 0x2c, 0xf3, 0x39, 0x94, 0xf4, 0x72, 0x0e, 0x00, 0x94, 0x13, 0x7c, 0x42],
+    [0x4f, 0xaa, 0x54, 0x1d, 0x5d, 0x49, 0x8e, 0x89, 0xba, 0x0e, 0xa4, 0xc3, 0x87, 0xb2, 0x2f, 0xb4],
+    [0x72, 0x3b, 0x9a, 0xf3, 0x55, 0x44, 0x91, 0xdb, 0xb1, 0xd6, 0x63, 0x3d, 0xfc, 0x6e, 0x0c, 0x4e],
+    [0xe5, 0x3f, 0x92, 0x85, 0x9e, 0x48, 0x19, 0xa8, 0xdc, 0x06, 0x95, 0x73, 0x9f, 0xea, 0x8c, 0x65],
+    [0xb2, 0xf8, 0x58, 0xc7, 0xc9, 0xea, 0x80, 0x1d, 0x53, 0xd6, 0x03, 0x59, 0x6d, 0x65, 0x78, 0x44],
+    [0x87, 0xe7, 0x62, 0x68, 0xdb, 0xc9, 0x22, 0x72, 0x26, 0xb0, 0xca, 0x66, 0x5f, 0x64, 0xe3, 0x78],
+    [0xc1, 0x7e, 0x55, 0x05, 0xb2, 0xbd, 0x52, 0x6c, 0x29, 0x21, 0xcd, 0xec, 0x1e, 0x7e, 0x01, 0x09],
+    [0xd0, 0xa8, 0xd9, 0x57, 0x15, 0x51, 0x8e, 0xeb, 0xb5, 0x13, 0xb0, 0xf8, 0x3d, 0x9e, 0x17, 0x93],
+    [0x23, 0x41, 0x26, 0xf9, 0x3f, 0xbb, 0x66, 0x8d, 0x97, 0x51, 0x12, 0xe8, 0xfe, 0xbd, 0xf7, 0xec],
+    [0xef, 0x42, 0xf0, 0x3d, 0xb7, 0x8f, 0x70, 0x4d, 0x02, 0x3c, 0x44, 0x9f, 0x16, 0xb7, 0x09, 0x2b],
+    [0xab, 0xf7, 0x62, 0x38, 0xc2, 0x0a, 0xf1, 0x61, 0xb2, 0x31, 0x4b, 0x4d, 0x55, 0x26, 0xbc, 0xe9],
+    [0x3c, 0x2c, 0x2f, 0x11, 0xbb, 0x90, 0xcf, 0x0b, 0xe3, 0x35, 0xca, 0x9b, 0x2e, 0x91, 0xe9, 0xb7],
+    [0x2a, 0x7a, 0x68, 0x0f, 0x22, 0xa0, 0x2a, 0x92, 0xf4, 0x51, 0x49, 0xd2, 0x0f, 0xec, 0xe0, 0xef],
+    [0xc9, 0xa8, 0xd1, 0x30, 0x23, 0x1d, 0xd4, 0x3e, 0x42, 0xe6, 0x45, 0x69, 0x57, 0xf8, 0x37, 0x79],
+    [0x1d, 0x12, 0x7b, 0x84, 0x40, 0x5c, 0xea, 0xb9, 0x9f, 0xd8, 0x77, 0x5a, 0x9b, 0xe6, 0xc5, 0x59],
+    [0x9e, 0x4b, 0xf8, 0x37, 0xbc, 0xfd, 0x92, 0xca, 0xce, 0x09, 0xd2, 0x06, 0x1a, 0x84, 0xd0, 0x4a],
+    [0x39, 0x03, 0x1a, 0x96, 0x5d, 0x73, 0xb4, 0xaf, 0x5a, 0x27, 0x4d, 0x18, 0xf9, 0x73, 0xb1, 0xd2],
+    [0x7f, 0x4d, 0x0a, 0x12, 0x09, 0xd6, 0x7e, 0x4e, 0xd0, 0x6f, 0x75, 0x38, 0xe1, 0xcf, 0xad, 0x64],
+    [0xe6, 0x1e, 0xe2, 0x40, 0xfb, 0xdc, 0xce, 0x38, 0x96, 0x9f, 0x4c, 0xd2, 0x49, 0x27, 0xdd, 0x93],
+    [0x4c, 0x3b, 0xa2, 0xb3, 0x7b, 0x0f, 0xdd, 0x8c, 0xfa, 0x5e, 0x95, 0xc1, 0x89, 0xb2, 0x94, 0x14],
+    [0xe0, 0x6f, 0xd4, 0xca, 0x06, 0x6f, 0xec, 0xdd, 0x54, 0x06, 0x8a, 0x5a, 0xd8, 0x89, 0x6f, 0x86],
+    [0x5c, 0xa8, 0x4c, 0x34, 0x13, 0x9c, 0x65, 0x80, 0xa8, 0x8a, 0xf2, 0x49, 0x90, 0x72, 0x07, 0x06],
+    [0x42, 0xea, 0x96, 0x1c, 0x5b, 0x3c, 0x85, 0x8b, 0x17, 0xc3, 0xe5, 0x50, 0xdf, 0xa7, 0x90, 0x10],
+    [0x40, 0x6c, 0x44, 0xde, 0xe6, 0x78, 0x57, 0xb2, 0x94, 0x31, 0x60, 0xf3, 0x0c, 0x74, 0x17, 0xd3],
+    [0xc5, 0xf5, 0x7b, 0xae, 0x13, 0x20, 0xfc, 0xf4, 0xb4, 0xe8, 0x68, 0xe7, 0x1d, 0x56, 0xc6, 0x6b],
+    [0x04, 0xbf, 0x73, 0x7a, 0x5b, 0x67, 0x6b, 0xe7, 0xc3, 0xde, 0x05, 0x01, 0x7d, 0xf4, 0xbf, 0xf9],
+    [0x51, 0x63, 0xc9, 0xc0, 0x3f, 0x19, 0x07, 0xea, 0x10, 0x44, 0xed, 0x5c, 0x30, 0x72, 0x7b, 0x4f],
+    [0x37, 0xa1, 0x10, 0xf0, 0x02, 0x71, 0x8e, 0xda, 0xd2, 0x4b, 0x3f, 0x9e, 0xe4, 0x53, 0xf1, 0x40],
+    [0xb9, 0x87, 0x7e, 0x38, 0x1a, 0xed, 0xd3, 0xda, 0x08, 0xc3, 0x3e, 0x75, 0xff, 0x23, 0xac, 0x10],
+    [0x7c, 0x50, 0x04, 0x00, 0x5e, 0xc5, 0xda, 0x4c, 0x5a, 0xc9, 0x44, 0x0e, 0x5c, 0x72, 0x31, 0x93],
+    [0x81, 0xb8, 0x24, 0x37, 0x83, 0xdb, 0xc6, 0x46, 0xca, 0x9d, 0x0c, 0xd8, 0x2a, 0xbd, 0xb4, 0x6c],
+    [0x50, 0x57, 0x20, 0x54, 0x3e, 0xb9, 0xb4, 0x13, 0xd5, 0x0b, 0x3c, 0xfa, 0xd9, 0xee, 0xf9, 0x38],
+    [0x94, 0x5f, 0x59, 0x4d, 0xe7, 0x24, 0x11, 0xe4, 0xd3, 0x35, 0xbe, 0x87, 0x44, 0x56, 0xd8, 0xf3],
+    [0x37, 0x92, 0x3b, 0x3e, 0x37, 0x17, 0x77, 0xb2, 0x11, 0x70, 0xbf, 0x9d, 0x7e, 0x62, 0xf6, 0x02],
+    [0x3a, 0xd4, 0xe7, 0xc8, 0x57, 0x64, 0x96, 0x46, 0x11, 0xeb, 0x0a, 0x6c, 0x4d, 0x62, 0xde, 0x56],
+    [0xcd, 0x91, 0x39, 0x6c, 0x44, 0xaf, 0x4f, 0x51, 0x85, 0x57, 0x8d, 0x9d, 0xd9, 0x80, 0x3f, 0x0a],
+    [0xfe, 0x28, 0x15, 0x8e, 0x72, 0x7b, 0x86, 0x8f, 0x39, 0x03, 0xc9, 0xac, 0xda, 0x64, 0xa2, 0x58],
+    [0x40, 0xcc, 0x10, 0xb8, 0x28, 0x8c, 0xe5, 0xf0, 0xbc, 0x3a, 0xc0, 0xb6, 0x8a, 0x0e, 0xeb, 0xc8],
+    [0x6f, 0x14, 0x90, 0xf5, 0x40, 0x69, 0x9a, 0x3c, 0xd4, 0x97, 0x44, 0x20, 0xec, 0xc9, 0x27, 0x37],
+    [0xd5, 0x05, 0xf1, 0xb7, 0x5e, 0x1a, 0x84, 0xa6, 0x03, 0xc4, 0x35, 0x83, 0xb2, 0xed, 0x03, 0x08],
+    [0x49, 0x15, 0x73, 0xcf, 0xd7, 0x2b, 0xb4, 0x68, 0x2b, 0x7c, 0xa5, 0x88, 0x0e, 0x1c, 0x8d, 0x6f],
+    [0x3e, 0xd6, 0x9c, 0xfe, 0x45, 0xab, 0x40, 0x3f, 0x2f, 0xd2, 0xad, 0x95, 0x9b, 0xa2, 0x76, 0x66],
+    [0x8b, 0xe8, 0x39, 0xef, 0x1b, 0x20, 0xb5, 0x7c, 0x83, 0xba, 0x7e, 0xb6, 0xa8, 0xc2, 0x2b, 0x6a],
+    [0x14, 0x09, 0x18, 0x6a, 0xb4, 0x22, 0x31, 0xfe, 0xde, 0xe1, 0x81, 0x62, 0xcf, 0x1c, 0xb4, 0xca],
+    [0x2b, 0xf3, 0xcc, 0xc2, 0x4a, 0xb6, 0x72, 0xcf, 0x15, 0x1f, 0xb8, 0xd2, 0xf3, 0xf3, 0x06, 0x9b],
+    [0xb9, 0xb9, 0x3a, 0x28, 0x82, 0xd6, 0x02, 0x5c, 0xdb, 0x8c, 0x56, 0xfa, 0x13, 0xf7, 0x53, 0x7b],
+    [0xd9, 0x7c, 0xca, 0x36, 0x94, 0xfb, 0x20, 0x6d, 0xb8, 0xbd, 0x1f, 0x36, 0x50, 0xc3, 0x33, 0x22],
+    [0x94, 0xec, 0x2e, 0x19, 0xa4, 0x0b, 0xe4, 0x1a, 0xf3, 0x94, 0x0d, 0x6b, 0x30, 0xc4, 0x93, 0x84],
+    [0x4b, 0x41, 0x60, 0x3f, 0x20, 0x9a, 0x04, 0x5b, 0xe1, 0x40, 0xa3, 0x41, 0xa3, 0xdf, 0xfe, 0x10],
+    [0x23, 0xfb, 0xcb, 0x30, 0x9f, 0x1c, 0xf0, 0x94, 0x89, 0x07, 0x55, 0xab, 0x1b, 0x42, 0x65, 0x69],
+    [0xe7, 0xd9, 0xb6, 0x56, 0x90, 0x91, 0x8a, 0x2b, 0x23, 0x2f, 0x2f, 0x5c, 0x12, 0xc8, 0x30, 0x0e],
+    [0xad, 0xe8, 0x3c, 0xf7, 0xe7, 0xf3, 0x84, 0x7b, 0x36, 0xfa, 0x4b, 0x54, 0xb0, 0x0d, 0xce, 0x61],
+    [0x06, 0x10, 0xc5, 0xf2, 0xee, 0x57, 0x1c, 0x8a, 0xc8, 0x0c, 0xbf, 0xe5, 0x38, 0xbd, 0xf1, 0xc7],
+    [0x27, 0x1d, 0x5d, 0x00, 0xfb, 0xdb, 0x5d, 0x15, 0x5d, 0x9d, 0xce, 0xa9, 0x7c, 0xb4, 0x02, 0x18],
+    [0x4c, 0x58, 0x00, 0xe3, 0x4e, 0xfe, 0x42, 0x6f, 0x07, 0x9f, 0x6b, 0x0a, 0xa7, 0x52, 0x60, 0xad],
 ];
 
-// Test vector from reference implementation
 #[test]
-fn test_siphash_2_4_test_vector() {
+fn test_siphash_1_3_test_vector() {
     let k0 = 0x_07_06_05_04_03_02_01_00;
     let k1 = 0x_0f_0e_0d_0c_0b_0a_09_08;
 
diff --git a/compiler/rustc_data_structures/src/sso/either_iter.rs b/compiler/rustc_data_structures/src/sso/either_iter.rs
deleted file mode 100644
index bca6c0955..000000000
--- a/compiler/rustc_data_structures/src/sso/either_iter.rs
+++ /dev/null
@@ -1,73 +0,0 @@
-use std::fmt;
-use std::iter::FusedIterator;
-
-/// Iterator which may contain instance of
-/// one of two specific implementations.
-///
-/// Note: For most methods providing custom
-///       implementation may marginally
-///       improve performance by avoiding
-///       doing Left/Right match on every step
-///       and doing it only once instead.
-#[derive(Clone)]
-pub enum EitherIter<L, R> {
-    Left(L),
-    Right(R),
-}
-
-impl<L, R> Iterator for EitherIter<L, R>
-where
-    L: Iterator,
-    R: Iterator<Item = L::Item>,
-{
-    type Item = L::Item;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self {
-            EitherIter::Left(l) => l.next(),
-            EitherIter::Right(r) => r.next(),
-        }
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        match self {
-            EitherIter::Left(l) => l.size_hint(),
-            EitherIter::Right(r) => r.size_hint(),
-        }
-    }
-}
-
-impl<L, R> ExactSizeIterator for EitherIter<L, R>
-where
-    L: ExactSizeIterator,
-    R: ExactSizeIterator,
-    EitherIter<L, R>: Iterator,
-{
-    fn len(&self) -> usize {
-        match self {
-            EitherIter::Left(l) => l.len(),
-            EitherIter::Right(r) => r.len(),
-        }
-    }
-}
-
-impl<L, R> FusedIterator for EitherIter<L, R>
-where
-    L: FusedIterator,
-    R: FusedIterator,
-    EitherIter<L, R>: Iterator,
-{
-}
-
-impl<L, R> fmt::Debug for EitherIter<L, R>
-where
-    L: fmt::Debug,
-    R: fmt::Debug,
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            EitherIter::Left(l) => l.fmt(f),
-            EitherIter::Right(r) => r.fmt(f),
-        }
-    }
-}
diff --git a/compiler/rustc_data_structures/src/sso/map.rs b/compiler/rustc_data_structures/src/sso/map.rs
index 7cdac5819..89b8c8526 100644
--- a/compiler/rustc_data_structures/src/sso/map.rs
+++ b/compiler/rustc_data_structures/src/sso/map.rs
@@ -1,24 +1,24 @@
-use super::either_iter::EitherIter;
 use crate::fx::FxHashMap;
 use arrayvec::ArrayVec;
+use itertools::Either;
 use std::fmt;
 use std::hash::Hash;
 use std::ops::Index;
 
-// For pointer-sized arguments arrays
-// are faster than set/map for up to 64
-// arguments.
-//
-// On the other hand such a big array
-// hurts cache performance, makes passing
-// sso structures around very expensive.
-//
-// Biggest performance benefit is gained
-// for reasonably small arrays that stay
-// small in vast majority of cases.
-//
-// '8' is chosen as a sane default, to be
-// reevaluated later.
+/// For pointer-sized arguments arrays
+/// are faster than set/map for up to 64
+/// arguments.
+///
+/// On the other hand such a big array
+/// hurts cache performance, makes passing
+/// sso structures around very expensive.
+///
+/// Biggest performance benefit is gained
+/// for reasonably small arrays that stay
+/// small in vast majority of cases.
+///
+/// '8' is chosen as a sane default, to be
+/// reevaluated later.
 const SSO_ARRAY_SIZE: usize = 8;
 
 /// Small-storage-optimized implementation of a map.
@@ -138,8 +138,8 @@ impl<K, V> SsoHashMap<K, V> {
     /// The iterator element type is `&'a K`.
     pub fn keys(&self) -> impl Iterator<Item = &'_ K> {
         match self {
-            SsoHashMap::Array(array) => EitherIter::Left(array.iter().map(|(k, _v)| k)),
-            SsoHashMap::Map(map) => EitherIter::Right(map.keys()),
+            SsoHashMap::Array(array) => Either::Left(array.iter().map(|(k, _v)| k)),
+            SsoHashMap::Map(map) => Either::Right(map.keys()),
         }
     }
 
@@ -147,8 +147,8 @@ impl<K, V> SsoHashMap<K, V> {
     /// The iterator element type is `&'a V`.
     pub fn values(&self) -> impl Iterator<Item = &'_ V> {
         match self {
-            SsoHashMap::Array(array) => EitherIter::Left(array.iter().map(|(_k, v)| v)),
-            SsoHashMap::Map(map) => EitherIter::Right(map.values()),
+            SsoHashMap::Array(array) => Either::Left(array.iter().map(|(_k, v)| v)),
+            SsoHashMap::Map(map) => Either::Right(map.values()),
         }
     }
 
@@ -156,8 +156,8 @@ impl<K, V> SsoHashMap<K, V> {
     /// The iterator element type is `&'a mut V`.
     pub fn values_mut(&mut self) -> impl Iterator<Item = &'_ mut V> {
         match self {
-            SsoHashMap::Array(array) => EitherIter::Left(array.iter_mut().map(|(_k, v)| v)),
-            SsoHashMap::Map(map) => EitherIter::Right(map.values_mut()),
+            SsoHashMap::Array(array) => Either::Left(array.iter_mut().map(|(_k, v)| v)),
+            SsoHashMap::Map(map) => Either::Right(map.values_mut()),
         }
     }
 
@@ -165,8 +165,8 @@ impl<K, V> SsoHashMap<K, V> {
     /// allocated memory for reuse.
     pub fn drain(&mut self) -> impl Iterator<Item = (K, V)> + '_ {
         match self {
-            SsoHashMap::Array(array) => EitherIter::Left(array.drain(..)),
-            SsoHashMap::Map(map) => EitherIter::Right(map.drain()),
+            SsoHashMap::Array(array) => Either::Left(array.drain(..)),
+            SsoHashMap::Map(map) => Either::Right(map.drain()),
         }
     }
 }
@@ -406,16 +406,16 @@ where
 }
 
 impl<K, V> IntoIterator for SsoHashMap<K, V> {
-    type IntoIter = EitherIter<
-        <ArrayVec<(K, V), 8> as IntoIterator>::IntoIter,
+    type IntoIter = Either<
+        <ArrayVec<(K, V), SSO_ARRAY_SIZE> as IntoIterator>::IntoIter,
         <FxHashMap<K, V> as IntoIterator>::IntoIter,
     >;
     type Item = <Self::IntoIter as Iterator>::Item;
 
     fn into_iter(self) -> Self::IntoIter {
         match self {
-            SsoHashMap::Array(array) => EitherIter::Left(array.into_iter()),
-            SsoHashMap::Map(map) => EitherIter::Right(map.into_iter()),
+            SsoHashMap::Array(array) => Either::Left(array.into_iter()),
+            SsoHashMap::Map(map) => Either::Right(map.into_iter()),
         }
     }
 }
@@ -435,9 +435,9 @@ fn adapt_array_mut_it<K, V>(pair: &mut (K, V)) -> (&K, &mut V) {
 }
 
 impl<'a, K, V> IntoIterator for &'a SsoHashMap<K, V> {
-    type IntoIter = EitherIter<
+    type IntoIter = Either<
         std::iter::Map<
-            <&'a ArrayVec<(K, V), 8> as IntoIterator>::IntoIter,
+            <&'a ArrayVec<(K, V), SSO_ARRAY_SIZE> as IntoIterator>::IntoIter,
             fn(&'a (K, V)) -> (&'a K, &'a V),
         >,
         <&'a FxHashMap<K, V> as IntoIterator>::IntoIter,
@@ -446,16 +446,16 @@ impl<'a, K, V> IntoIterator for &'a SsoHashMap<K, V> {
 
     fn into_iter(self) -> Self::IntoIter {
         match self {
-            SsoHashMap::Array(array) => EitherIter::Left(array.into_iter().map(adapt_array_ref_it)),
-            SsoHashMap::Map(map) => EitherIter::Right(map.iter()),
+            SsoHashMap::Array(array) => Either::Left(array.into_iter().map(adapt_array_ref_it)),
+            SsoHashMap::Map(map) => Either::Right(map.iter()),
         }
     }
 }
 
 impl<'a, K, V> IntoIterator for &'a mut SsoHashMap<K, V> {
-    type IntoIter = EitherIter<
+    type IntoIter = Either<
         std::iter::Map<
-            <&'a mut ArrayVec<(K, V), 8> as IntoIterator>::IntoIter,
+            <&'a mut ArrayVec<(K, V), SSO_ARRAY_SIZE> as IntoIterator>::IntoIter,
             fn(&'a mut (K, V)) -> (&'a K, &'a mut V),
         >,
         <&'a mut FxHashMap<K, V> as IntoIterator>::IntoIter,
@@ -464,8 +464,8 @@ impl<'a, K, V> IntoIterator for &'a mut SsoHashMap<K, V> {
 
     fn into_iter(self) -> Self::IntoIter {
         match self {
-            SsoHashMap::Array(array) => EitherIter::Left(array.into_iter().map(adapt_array_mut_it)),
-            SsoHashMap::Map(map) => EitherIter::Right(map.iter_mut()),
+            SsoHashMap::Array(array) => Either::Left(array.into_iter().map(adapt_array_mut_it)),
+            SsoHashMap::Map(map) => Either::Right(map.iter_mut()),
         }
     }
 }
diff --git a/compiler/rustc_data_structures/src/sso/mod.rs b/compiler/rustc_data_structures/src/sso/mod.rs
index dd21bc8e6..ef634b9ad 100644
--- a/compiler/rustc_data_structures/src/sso/mod.rs
+++ b/compiler/rustc_data_structures/src/sso/mod.rs
@@ -1,4 +1,3 @@
-mod either_iter;
 mod map;
 mod set;
 
diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs
index e0d77cdae..3ed1de1bc 100644
--- a/compiler/rustc_data_structures/src/stable_hasher.rs
+++ b/compiler/rustc_data_structures/src/stable_hasher.rs
@@ -312,14 +312,14 @@ impl<CTX> HashStable<CTX> for ::std::num::NonZeroUsize {
 
 impl<CTX> HashStable<CTX> for f32 {
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
-        let val: u32 = unsafe { ::std::mem::transmute(*self) };
+        let val: u32 = self.to_bits();
         val.hash_stable(ctx, hasher);
     }
 }
 
 impl<CTX> HashStable<CTX> for f64 {
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
-        let val: u64 = unsafe { ::std::mem::transmute(*self) };
+        let val: u64 = self.to_bits();
         val.hash_stable(ctx, hasher);
     }
 }
@@ -617,18 +617,10 @@ where
     }
 }
 
-impl<K, R, HCX> HashStable<HCX> for ::std::collections::HashSet<K, R>
-where
-    K: ToStableHashKey<HCX> + Eq,
-    R: BuildHasher,
-{
-    fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
-        stable_hash_reduce(hcx, hasher, self.iter(), self.len(), |hasher, hcx, key| {
-            let key = key.to_stable_hash_key(hcx);
-            key.hash_stable(hcx, hasher);
-        });
-    }
-}
+// It is not safe to implement HashStable for HashSet or any other collection type
+// with unstable but observable iteration order.
+// See https://github.com/rust-lang/compiler-team/issues/533 for further information.
+impl<V, HCX> !HashStable<HCX> for std::collections::HashSet<V> {}
 
 impl<K, V, HCX> HashStable<HCX> for ::std::collections::BTreeMap<K, V>
 where
diff --git a/compiler/rustc_data_structures/src/stable_hasher/tests.rs b/compiler/rustc_data_structures/src/stable_hasher/tests.rs
index 724be5888..a98b1bc36 100644
--- a/compiler/rustc_data_structures/src/stable_hasher/tests.rs
+++ b/compiler/rustc_data_structures/src/stable_hasher/tests.rs
@@ -39,7 +39,7 @@ fn test_hash_integers() {
     test_isize.hash(&mut h);
 
     // This depends on the hashing algorithm. See note at top of file.
-    let expected = (1784307454142909076, 11471672289340283879);
+    let expected = (13997337031081104755, 6178945012502239489);
 
     assert_eq!(h.finalize(), expected);
 }
@@ -53,7 +53,7 @@ fn test_hash_usize() {
     test_usize.hash(&mut h);
 
     // This depends on the hashing algorithm. See note at top of file.
-    let expected = (5798740672699530587, 11186240177685111648);
+    let expected = (12037165114281468837, 3094087741167521712);
 
     assert_eq!(h.finalize(), expected);
 }
@@ -67,7 +67,7 @@ fn test_hash_isize() {
     test_isize.hash(&mut h);
 
     // This depends on the hashing algorithm. See note at top of file.
-    let expected = (2789913510339652884, 674280939192711005);
+    let expected = (3979067582695659080, 2322428596355037273);
 
     assert_eq!(h.finalize(), expected);
 }
diff --git a/compiler/rustc_data_structures/src/stack.rs b/compiler/rustc_data_structures/src/stack.rs
index 3bdd67512..7ff1339c5 100644
--- a/compiler/rustc_data_structures/src/stack.rs
+++ b/compiler/rustc_data_structures/src/stack.rs
@@ -5,7 +5,7 @@ const RED_ZONE: usize = 100 * 1024; // 100k
 
 // Only the first stack that is pushed, grows exponentially (2^n * STACK_PER_RECURSION) from then
 // on. This flag has performance relevant characteristics. Don't set it too high.
-const STACK_PER_RECURSION: usize = 1 * 1024 * 1024; // 1MB
+const STACK_PER_RECURSION: usize = 1024 * 1024; // 1MB
 
 /// Grows the stack on demand to prevent stack overflow. Call this in strategic locations
 /// to "break up" recursive calls. E.g. almost any call to `visit_expr` or equivalent can benefit
diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs
index 31323c21d..ef1da8519 100644
--- a/compiler/rustc_data_structures/src/sync.rs
+++ b/compiler/rustc_data_structures/src/sync.rs
@@ -1,23 +1,45 @@
-//! This module defines types which are thread safe if cfg!(parallel_compiler) is true.
+//! This module defines various operations and types that are implemented in
+//! one way for the serial compiler, and another way the parallel compiler.
 //!
-//! `Lrc` is an alias of `Arc` if cfg!(parallel_compiler) is true, `Rc` otherwise.
+//! Operations
+//! ----------
+//! The parallel versions of operations use Rayon to execute code in parallel,
+//! while the serial versions degenerate straightforwardly to serial execution.
+//! The operations include `join`, `parallel`, `par_iter`, and `par_for_each`.
 //!
-//! `Lock` is a mutex.
-//! It internally uses `parking_lot::Mutex` if cfg!(parallel_compiler) is true,
-//! `RefCell` otherwise.
+//! Types
+//! -----
+//! The parallel versions of types provide various kinds of synchronization,
+//! while the serial compiler versions do not.
 //!
-//! `RwLock` is a read-write lock.
-//! It internally uses `parking_lot::RwLock` if cfg!(parallel_compiler) is true,
-//! `RefCell` otherwise.
+//! The following table shows how the types are implemented internally. Except
+//! where noted otherwise, the type in column one is defined as a
+//! newtype around the type from column two or three.
 //!
-//! `MTLock` is a mutex which disappears if cfg!(parallel_compiler) is false.
+//! | Type                    | Serial version      | Parallel version                |
+//! | ----------------------- | ------------------- | ------------------------------- |
+//! | `Lrc<T>`                | `rc::Rc<T>`         | `sync::Arc<T>`                  |
+//! |` Weak<T>`               | `rc::Weak<T>`       | `sync::Weak<T>`                 |
+//! |                         |                     |                                 |
+//! | `AtomicBool`            | `Cell<bool>`        | `atomic::AtomicBool`            |
+//! | `AtomicU32`             | `Cell<u32>`         | `atomic::AtomicU32`             |
+//! | `AtomicU64`             | `Cell<u64>`         | `atomic::AtomicU64`             |
+//! | `AtomicUsize`           | `Cell<usize>`       | `atomic::AtomicUsize`           |
+//! |                         |                     |                                 |
+//! | `Lock<T>`               | `RefCell<T>`        | `parking_lot::Mutex<T>`         |
+//! | `RwLock<T>`             | `RefCell<T>`        | `parking_lot::RwLock<T>`        |
+//! | `MTLock<T>`        [^1] | `T`                 | `Lock<T>`                       |
+//! | `MTLockRef<'a, T>` [^2] | `&'a mut MTLock<T>` | `&'a MTLock<T>`                 |
+//! |                         |                     |                                 |
+//! | `ParallelIterator`      | `Iterator`          | `rayon::iter::ParallelIterator` |
 //!
-//! `MTRef` is an immutable reference if cfg!(parallel_compiler), and a mutable reference otherwise.
+//! [^1] `MTLock` is similar to `Lock`, but the serial version avoids the cost
+//! of a `RefCell`. This is appropriate when interior mutability is not
+//! required.
 //!
-//! `rustc_erase_owner!` erases an OwningRef owner into Erased or Erased + Send + Sync
-//! depending on the value of cfg!(parallel_compiler).
+//! [^2] `MTLockRef` is a typedef.
 
-use crate::owning_ref::{Erased, OwningRef};
+use crate::owned_slice::OwnedSlice;
 use std::collections::HashMap;
 use std::hash::{BuildHasher, Hash};
 use std::ops::{Deref, DerefMut};
@@ -26,24 +48,17 @@ use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
 pub use std::sync::atomic::Ordering;
 pub use std::sync::atomic::Ordering::SeqCst;
 
-pub use vec::AppendOnlyVec;
+pub use vec::{AppendOnlyIndexVec, AppendOnlyVec};
 
 mod vec;
 
 cfg_if! {
     if #[cfg(not(parallel_compiler))] {
-        pub auto trait Send {}
-        pub auto trait Sync {}
+        pub unsafe auto trait Send {}
+        pub unsafe auto trait Sync {}
 
-        impl<T> Send for T {}
-        impl<T> Sync for T {}
-
-        #[macro_export]
-        macro_rules! rustc_erase_owner {
-            ($v:expr) => {
-                $v.erase_owner()
-            }
-        }
+        unsafe impl<T> Send for T {}
+        unsafe impl<T> Sync for T {}
 
         use std::ops::Add;
 
@@ -82,6 +97,14 @@ cfg_if! {
             }
         }
 
+        impl Atomic<bool> {
+            pub fn fetch_or(&self, val: bool, _: Ordering) -> bool {
+                let result = self.0.get() | val;
+                self.0.set(val);
+                result
+            }
+        }
+
         impl<T: Copy + PartialEq> Atomic<T> {
             #[inline]
             pub fn compare_exchange(&self,
@@ -164,7 +187,7 @@ cfg_if! {
             }
         }
 
-        pub type MetadataRef = OwningRef<Box<dyn Erased>, [u8]>;
+        pub type MetadataRef = OwnedSlice;
 
         pub use std::rc::Rc as Lrc;
         pub use std::rc::Weak as Weak;
@@ -209,7 +232,7 @@ cfg_if! {
             }
         }
 
-        pub type MTRef<'a, T> = &'a mut T;
+        pub type MTLockRef<'a, T> = &'a mut MTLock<T>;
 
         #[derive(Debug, Default)]
         pub struct MTLock<T>(T);
@@ -267,7 +290,7 @@ cfg_if! {
         pub use std::sync::Arc as Lrc;
         pub use std::sync::Weak as Weak;
 
-        pub type MTRef<'a, T> = &'a T;
+        pub type MTLockRef<'a, T> = &'a MTLock<T>;
 
         #[derive(Debug, Default)]
         pub struct MTLock<T>(Lock<T>);
@@ -347,20 +370,11 @@ cfg_if! {
             });
         }
 
-        pub type MetadataRef = OwningRef<Box<dyn Erased + Send + Sync>, [u8]>;
+        pub type MetadataRef = OwnedSlice;
 
         /// This makes locks panic if they are already held.
         /// It is only useful when you are running in a single thread
         const ERROR_CHECKING: bool = false;
-
-        #[macro_export]
-        macro_rules! rustc_erase_owner {
-            ($v:expr) => {{
-                let v = $v;
-                ::rustc_data_structures::sync::assert_send_val(&v);
-                v.erase_send_sync_owner()
-            }}
-        }
     }
 }
 
@@ -456,14 +470,6 @@ impl<T: Default> Default for Lock<T> {
     }
 }
 
-// FIXME: Probably a bad idea
-impl<T: Clone> Clone for Lock<T> {
-    #[inline]
-    fn clone(&self) -> Self {
-        Lock::new(self.borrow().clone())
-    }
-}
-
 #[derive(Debug, Default)]
 pub struct RwLock<T>(InnerRwLock<T>);
 
@@ -555,18 +561,6 @@ impl<T> RwLock<T> {
 
     #[cfg(not(parallel_compiler))]
     #[inline(always)]
-    pub fn clone_guard<'a>(rg: &ReadGuard<'a, T>) -> ReadGuard<'a, T> {
-        ReadGuard::clone(rg)
-    }
-
-    #[cfg(parallel_compiler)]
-    #[inline(always)]
-    pub fn clone_guard<'a>(rg: &ReadGuard<'a, T>) -> ReadGuard<'a, T> {
-        ReadGuard::rwlock(&rg).read()
-    }
-
-    #[cfg(not(parallel_compiler))]
-    #[inline(always)]
     pub fn leak(&self) -> &T {
         ReadGuard::leak(self.read())
     }
diff --git a/compiler/rustc_data_structures/src/sync/vec.rs b/compiler/rustc_data_structures/src/sync/vec.rs
index cbea4f059..1783b4b35 100644
--- a/compiler/rustc_data_structures/src/sync/vec.rs
+++ b/compiler/rustc_data_structures/src/sync/vec.rs
@@ -2,7 +2,8 @@ use std::marker::PhantomData;
 
 use rustc_index::vec::Idx;
 
-pub struct AppendOnlyVec<I: Idx, T: Copy> {
+#[derive(Default)]
+pub struct AppendOnlyIndexVec<I: Idx, T: Copy> {
     #[cfg(not(parallel_compiler))]
     vec: elsa::vec::FrozenVec<T>,
     #[cfg(parallel_compiler)]
@@ -10,7 +11,7 @@ pub struct AppendOnlyVec<I: Idx, T: Copy> {
     _marker: PhantomData<fn(&I)>,
 }
 
-impl<I: Idx, T: Copy> AppendOnlyVec<I, T> {
+impl<I: Idx, T: Copy> AppendOnlyIndexVec<I, T> {
     pub fn new() -> Self {
         Self {
             #[cfg(not(parallel_compiler))]
@@ -39,3 +40,66 @@ impl<I: Idx, T: Copy> AppendOnlyVec<I, T> {
         return self.vec.get(i);
     }
 }
+
+#[derive(Default)]
+pub struct AppendOnlyVec<T: Copy> {
+    #[cfg(not(parallel_compiler))]
+    vec: elsa::vec::FrozenVec<T>,
+    #[cfg(parallel_compiler)]
+    vec: elsa::sync::LockFreeFrozenVec<T>,
+}
+
+impl<T: Copy> AppendOnlyVec<T> {
+    pub fn new() -> Self {
+        Self {
+            #[cfg(not(parallel_compiler))]
+            vec: elsa::vec::FrozenVec::new(),
+            #[cfg(parallel_compiler)]
+            vec: elsa::sync::LockFreeFrozenVec::new(),
+        }
+    }
+
+    pub fn push(&self, val: T) -> usize {
+        #[cfg(not(parallel_compiler))]
+        let i = self.vec.len();
+        #[cfg(not(parallel_compiler))]
+        self.vec.push(val);
+        #[cfg(parallel_compiler)]
+        let i = self.vec.push(val);
+        i
+    }
+
+    pub fn get(&self, i: usize) -> Option<T> {
+        #[cfg(not(parallel_compiler))]
+        return self.vec.get_copy(i);
+        #[cfg(parallel_compiler)]
+        return self.vec.get(i);
+    }
+
+    pub fn iter_enumerated(&self) -> impl Iterator<Item = (usize, T)> + '_ {
+        (0..)
+            .map(|i| (i, self.get(i)))
+            .take_while(|(_, o)| o.is_some())
+            .filter_map(|(i, o)| Some((i, o?)))
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = T> + '_ {
+        (0..).map(|i| self.get(i)).take_while(|o| o.is_some()).flatten()
+    }
+}
+
+impl<T: Copy + PartialEq> AppendOnlyVec<T> {
+    pub fn contains(&self, val: T) -> bool {
+        self.iter_enumerated().any(|(_, v)| v == val)
+    }
+}
+
+impl<A: Copy> FromIterator<A> for AppendOnlyVec<A> {
+    fn from_iter<T: IntoIterator<Item = A>>(iter: T) -> Self {
+        let this = Self::new();
+        for val in iter {
+            this.push(val);
+        }
+        this
+    }
+}
diff --git a/compiler/rustc_data_structures/src/unord.rs b/compiler/rustc_data_structures/src/unord.rs
index f35f18e51..6c8d54146 100644
--- a/compiler/rustc_data_structures/src/unord.rs
+++ b/compiler/rustc_data_structures/src/unord.rs
@@ -109,6 +109,12 @@ impl<T, I: Iterator<Item = T>> UnordItems<T, I> {
     }
 }
 
+impl<T> UnordItems<T, std::iter::Empty<T>> {
+    pub fn empty() -> Self {
+        UnordItems(std::iter::empty())
+    }
+}
+
 impl<'a, T: Clone + 'a, I: Iterator<Item = &'a T>> UnordItems<&'a T, I> {
     #[inline]
     pub fn cloned(self) -> UnordItems<T, impl Iterator<Item = T>> {
@@ -133,6 +139,20 @@ impl<T: Ord, I: Iterator<Item = T>> UnordItems<T, I> {
         items
     }
 
+    #[inline]
+    pub fn into_sorted_stable_ord(self, use_stable_sort: bool) -> Vec<T>
+    where
+        T: Ord + StableOrd,
+    {
+        let mut items: Vec<T> = self.0.collect();
+        if use_stable_sort {
+            items.sort();
+        } else {
+            items.sort_unstable()
+        }
+        items
+    }
+
     pub fn into_sorted_small_vec<HCX, const LEN: usize>(self, hcx: &HCX) -> SmallVec<[T; LEN]>
     where
         T: ToStableHashKey<HCX>,
@@ -176,6 +196,11 @@ impl<V: Eq + Hash> UnordSet<V> {
     }
 
     #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.inner.is_empty()
+    }
+
+    #[inline]
     pub fn insert(&mut self, v: V) -> bool {
         self.inner.insert(v)
     }
@@ -199,7 +224,7 @@ impl<V: Eq + Hash> UnordSet<V> {
     }
 
     #[inline]
-    pub fn items<'a>(&'a self) -> UnordItems<&'a V, impl Iterator<Item = &'a V>> {
+    pub fn items(&self) -> UnordItems<&V, impl Iterator<Item = &V>> {
         UnordItems(self.inner.iter())
     }
 
@@ -253,7 +278,7 @@ impl<V: Eq + Hash> UnordSet<V> {
     // We can safely extend this UnordSet from a set of unordered values because that
     // won't expose the internal ordering anywhere.
     #[inline]
-    pub fn extend<I: Iterator<Item = V>>(&mut self, items: UnordItems<V, I>) {
+    pub fn extend_unord<I: Iterator<Item = V>>(&mut self, items: UnordItems<V, I>) {
         self.inner.extend(items.0)
     }
 
@@ -277,6 +302,12 @@ impl<V: Hash + Eq> FromIterator<V> for UnordSet<V> {
     }
 }
 
+impl<V: Hash + Eq> From<FxHashSet<V>> for UnordSet<V> {
+    fn from(value: FxHashSet<V>) -> Self {
+        UnordSet { inner: value }
+    }
+}
+
 impl<HCX, V: Hash + Eq + HashStable<HCX>> HashStable<HCX> for UnordSet<V> {
     #[inline]
     fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
@@ -384,7 +415,7 @@ impl<K: Eq + Hash, V> UnordMap<K, V> {
     }
 
     #[inline]
-    pub fn items<'a>(&'a self) -> UnordItems<(&'a K, &'a V), impl Iterator<Item = (&'a K, &'a V)>> {
+    pub fn items(&self) -> UnordItems<(&K, &V), impl Iterator<Item = (&K, &V)>> {
         UnordItems(self.inner.iter())
     }
 
diff --git a/compiler/rustc_data_structures/src/vec_map.rs b/compiler/rustc_data_structures/src/vec_map.rs
deleted file mode 100644
index d1a99bcae..000000000
--- a/compiler/rustc_data_structures/src/vec_map.rs
+++ /dev/null
@@ -1,192 +0,0 @@
-use std::borrow::Borrow;
-use std::fmt::Debug;
-use std::slice::Iter;
-use std::vec::IntoIter;
-
-use crate::stable_hasher::{HashStable, StableHasher};
-
-/// A map type implemented as a vector of pairs `K` (key) and `V` (value).
-/// It currently provides a subset of all the map operations, the rest could be added as needed.
-#[derive(Clone, Encodable, Decodable, Debug)]
-pub struct VecMap<K, V>(Vec<(K, V)>);
-
-impl<K, V> VecMap<K, V>
-where
-    K: Debug + PartialEq,
-    V: Debug,
-{
-    pub fn new() -> Self {
-        VecMap(Default::default())
-    }
-
-    /// Sets the value of the entry, and returns the entry's old value.
-    pub fn insert(&mut self, k: K, v: V) -> Option<V> {
-        if let Some(elem) = self.0.iter_mut().find(|(key, _)| *key == k) {
-            Some(std::mem::replace(&mut elem.1, v))
-        } else {
-            self.0.push((k, v));
-            None
-        }
-    }
-
-    /// Removes the entry from the map and returns the removed value
-    pub fn remove(&mut self, k: &K) -> Option<V> {
-        self.0.iter().position(|(k2, _)| k2 == k).map(|pos| self.0.remove(pos).1)
-    }
-
-    /// Gets a reference to the value in the entry.
-    pub fn get<Q: ?Sized>(&self, k: &Q) -> Option<&V>
-    where
-        K: Borrow<Q>,
-        Q: Eq,
-    {
-        self.0.iter().find(|(key, _)| k == key.borrow()).map(|elem| &elem.1)
-    }
-
-    /// Gets a mutable reference to the value in the entry.
-    pub fn get_mut<Q: ?Sized>(&mut self, k: &Q) -> Option<&mut V>
-    where
-        K: Borrow<Q>,
-        Q: Eq,
-    {
-        self.0.iter_mut().find(|(key, _)| k == key.borrow()).map(|elem| &mut elem.1)
-    }
-
-    /// Returns the any value corresponding to the supplied predicate filter.
-    ///
-    /// The supplied predicate will be applied to each (key, value) pair and it will return a
-    /// reference to the values where the predicate returns `true`.
-    pub fn any_value_matching(&self, mut predicate: impl FnMut(&(K, V)) -> bool) -> Option<&V> {
-        self.0.iter().find(|kv| predicate(kv)).map(|elem| &elem.1)
-    }
-
-    /// Returns the value corresponding to the supplied predicate filter. It crashes if there's
-    /// more than one matching element.
-    ///
-    /// The supplied predicate will be applied to each (key, value) pair and it will return a
-    /// reference to the value where the predicate returns `true`.
-    pub fn get_value_matching(&self, mut predicate: impl FnMut(&(K, V)) -> bool) -> Option<&V> {
-        let mut filter = self.0.iter().filter(|kv| predicate(kv));
-        let (_, value) = filter.next()?;
-        // This should return just one element, otherwise it's a bug
-        assert!(
-            filter.next().is_none(),
-            "Collection {self:#?} should have just one matching element"
-        );
-        Some(value)
-    }
-
-    /// Returns `true` if the map contains a value for the specified key.
-    ///
-    /// The key may be any borrowed form of the map's key type,
-    /// [`Eq`] on the borrowed form *must* match those for
-    /// the key type.
-    pub fn contains_key<Q: ?Sized>(&self, k: &Q) -> bool
-    where
-        K: Borrow<Q>,
-        Q: Eq,
-    {
-        self.get(k).is_some()
-    }
-
-    /// Returns `true` if the map contains no elements.
-    pub fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-
-    pub fn iter(&self) -> Iter<'_, (K, V)> {
-        self.into_iter()
-    }
-
-    pub fn iter_mut(&mut self) -> impl Iterator<Item = (&K, &mut V)> {
-        self.into_iter()
-    }
-
-    pub fn retain(&mut self, f: impl Fn(&(K, V)) -> bool) {
-        self.0.retain(f)
-    }
-}
-
-impl<K, V> Default for VecMap<K, V> {
-    #[inline]
-    fn default() -> Self {
-        Self(Default::default())
-    }
-}
-
-impl<K, V> From<Vec<(K, V)>> for VecMap<K, V> {
-    fn from(vec: Vec<(K, V)>) -> Self {
-        Self(vec)
-    }
-}
-
-impl<K, V> Into<Vec<(K, V)>> for VecMap<K, V> {
-    fn into(self) -> Vec<(K, V)> {
-        self.0
-    }
-}
-
-impl<K, V> FromIterator<(K, V)> for VecMap<K, V> {
-    fn from_iter<I: IntoIterator<Item = (K, V)>>(iter: I) -> Self {
-        Self(iter.into_iter().collect())
-    }
-}
-
-impl<'a, K, V> IntoIterator for &'a VecMap<K, V> {
-    type Item = &'a (K, V);
-    type IntoIter = Iter<'a, (K, V)>;
-
-    #[inline]
-    fn into_iter(self) -> Self::IntoIter {
-        self.0.iter()
-    }
-}
-
-impl<'a, K: 'a, V: 'a> IntoIterator for &'a mut VecMap<K, V> {
-    type Item = (&'a K, &'a mut V);
-    type IntoIter = impl Iterator<Item = Self::Item>;
-
-    #[inline]
-    fn into_iter(self) -> Self::IntoIter {
-        self.0.iter_mut().map(|(k, v)| (&*k, v))
-    }
-}
-
-impl<K, V> IntoIterator for VecMap<K, V> {
-    type Item = (K, V);
-    type IntoIter = IntoIter<(K, V)>;
-
-    #[inline]
-    fn into_iter(self) -> Self::IntoIter {
-        self.0.into_iter()
-    }
-}
-
-impl<K: PartialEq + Debug, V: Debug> Extend<(K, V)> for VecMap<K, V> {
-    fn extend<I: IntoIterator<Item = (K, V)>>(&mut self, iter: I) {
-        for (k, v) in iter {
-            self.insert(k, v);
-        }
-    }
-
-    fn extend_one(&mut self, (k, v): (K, V)) {
-        self.insert(k, v);
-    }
-
-    fn extend_reserve(&mut self, additional: usize) {
-        self.0.extend_reserve(additional);
-    }
-}
-
-impl<K, V, CTX> HashStable<CTX> for VecMap<K, V>
-where
-    K: HashStable<CTX> + Eq,
-    V: HashStable<CTX>,
-{
-    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
-        self.0.hash_stable(hcx, hasher)
-    }
-}
-
-#[cfg(test)]
-mod tests;
diff --git a/compiler/rustc_data_structures/src/vec_map/tests.rs b/compiler/rustc_data_structures/src/vec_map/tests.rs
deleted file mode 100644
index 458b60077..000000000
--- a/compiler/rustc_data_structures/src/vec_map/tests.rs
+++ /dev/null
@@ -1,48 +0,0 @@
-use super::*;
-
-impl<K, V> VecMap<K, V> {
-    fn into_vec(self) -> Vec<(K, V)> {
-        self.0.into()
-    }
-}
-
-#[test]
-fn test_from_iterator() {
-    assert_eq!(
-        std::iter::empty().collect::<VecMap<i32, bool>>().into_vec(),
-        Vec::<(i32, bool)>::new()
-    );
-    assert_eq!(std::iter::once((42, true)).collect::<VecMap<_, _>>().into_vec(), vec![(42, true)]);
-    assert_eq!(
-        [(1, true), (2, false)].into_iter().collect::<VecMap<_, _>>().into_vec(),
-        vec![(1, true), (2, false)]
-    );
-}
-
-#[test]
-fn test_into_iterator_owned() {
-    assert_eq!(VecMap::new().into_iter().collect::<Vec<(i32, bool)>>(), Vec::<(i32, bool)>::new());
-    assert_eq!(VecMap::from(vec![(1, true)]).into_iter().collect::<Vec<_>>(), vec![(1, true)]);
-    assert_eq!(
-        VecMap::from(vec![(1, true), (2, false)]).into_iter().collect::<Vec<_>>(),
-        vec![(1, true), (2, false)]
-    );
-}
-
-#[test]
-fn test_insert() {
-    let mut v = VecMap::new();
-    assert_eq!(v.insert(1, true), None);
-    assert_eq!(v.insert(2, false), None);
-    assert_eq!(v.clone().into_vec(), vec![(1, true), (2, false)]);
-    assert_eq!(v.insert(1, false), Some(true));
-    assert_eq!(v.into_vec(), vec![(1, false), (2, false)]);
-}
-
-#[test]
-fn test_get() {
-    let v = [(1, true), (2, false)].into_iter().collect::<VecMap<_, _>>();
-    assert_eq!(v.get(&1), Some(&true));
-    assert_eq!(v.get(&2), Some(&false));
-    assert_eq!(v.get(&3), None);
-}
diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml
index 7b59a52cf..73a1f79a0 100644
--- a/compiler/rustc_driver_impl/Cargo.toml
+++ b/compiler/rustc_driver_impl/Cargo.toml
@@ -54,8 +54,11 @@ rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 [target.'cfg(unix)'.dependencies]
 libc = "0.2"
 
-[target.'cfg(windows)'.dependencies]
-winapi = { version = "0.3", features = ["consoleapi", "debugapi", "processenv"] }
+[target.'cfg(windows)'.dependencies.windows]
+version = "0.46.0"
+features = [
+    "Win32_System_Diagnostics_Debug",
+]
 
 [features]
 llvm = ['rustc_interface/llvm']
diff --git a/compiler/rustc_driver_impl/locales/en-US.ftl b/compiler/rustc_driver_impl/messages.ftl
index f19b1ff64..f19b1ff64 100644
--- a/compiler/rustc_driver_impl/locales/en-US.ftl
+++ b/compiler/rustc_driver_impl/messages.ftl
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 464ddae47..b9f0e756e 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -5,8 +5,7 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![feature(is_terminal)]
-#![feature(once_cell)]
+#![feature(lazy_cell)]
 #![feature(decl_macro)]
 #![recursion_limit = "256"]
 #![allow(rustc::potential_query_instability)]
@@ -20,7 +19,9 @@ pub extern crate rustc_plugin_impl as plugin;
 
 use rustc_ast as ast;
 use rustc_codegen_ssa::{traits::CodegenBackend, CodegenErrors, CodegenResults};
-use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
+use rustc_data_structures::profiling::{
+    get_resident_set_size, print_time_passes_entry, TimePassesFormat,
+};
 use rustc_data_structures::sync::SeqCst;
 use rustc_errors::registry::{InvalidErrorCode, Registry};
 use rustc_errors::{
@@ -35,15 +36,17 @@ use rustc_metadata::locator;
 use rustc_session::config::{nightly_options, CG_OPTIONS, Z_OPTIONS};
 use rustc_session::config::{ErrorOutputType, Input, OutputType, PrintRequest, TrimmedDefPaths};
 use rustc_session::cstore::MetadataLoader;
-use rustc_session::getopts;
+use rustc_session::getopts::{self, Matches};
 use rustc_session::lint::{Lint, LintId};
 use rustc_session::{config, Session};
 use rustc_session::{early_error, early_error_no_abort, early_warn};
 use rustc_span::source_map::{FileLoader, FileName};
 use rustc_span::symbol::sym;
 use rustc_target::json::ToJson;
+use rustc_target::spec::{Target, TargetTriple};
 
 use std::cmp::max;
+use std::collections::BTreeMap;
 use std::env;
 use std::ffi::OsString;
 use std::fs;
@@ -64,7 +67,7 @@ use crate::session_diagnostics::{
     RLinkWrongFileType, RlinkNotAFile, RlinkUnableToRead,
 };
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[
     // tidy-alphabetical-start
@@ -161,7 +164,7 @@ pub trait Callbacks {
 
 #[derive(Default)]
 pub struct TimePassesCallbacks {
-    time_passes: bool,
+    time_passes: Option<TimePassesFormat>,
 }
 
 impl Callbacks for TimePassesCallbacks {
@@ -171,7 +174,8 @@ impl Callbacks for TimePassesCallbacks {
         // If a --print=... option has been given, we don't print the "total"
         // time because it will mess up the --print output. See #64339.
         //
-        self.time_passes = config.opts.prints.is_empty() && config.opts.unstable_opts.time_passes;
+        self.time_passes = (config.opts.prints.is_empty() && config.opts.unstable_opts.time_passes)
+            .then(|| config.opts.unstable_opts.time_passes_format);
         config.opts.trimmed_def_paths = TrimmedDefPaths::GoodPath;
     }
 }
@@ -331,6 +335,7 @@ fn run_compiler(
             if let Some(ppm) = &sess.opts.pretty {
                 if ppm.needs_ast_map() {
                     queries.global_ctxt()?.enter(|tcx| {
+                        tcx.ensure().early_lint_checks(());
                         pretty::print_after_hir_lowering(tcx, *ppm);
                         Ok(())
                     })?;
@@ -352,7 +357,7 @@ fn run_compiler(
 
             {
                 let plugins = queries.register_plugins()?;
-                let (_, lint_store) = &*plugins.borrow();
+                let (.., lint_store) = &*plugins.borrow();
 
                 // Lint plugins are registered; now we can process command line flags.
                 if sess.opts.describe_lints {
@@ -644,6 +649,15 @@ fn print_crate_info(
             TargetSpec => {
                 println!("{}", serde_json::to_string_pretty(&sess.target.to_json()).unwrap());
             }
+            AllTargetSpecs => {
+                let mut targets = BTreeMap::new();
+                for name in rustc_target::spec::TARGETS {
+                    let triple = TargetTriple::from_triple(name);
+                    let target = Target::expect_builtin(&triple);
+                    targets.insert(name, target.to_json());
+                }
+                println!("{}", serde_json::to_string_pretty(&targets).unwrap());
+            }
             FileNames | CrateName => {
                 let Some(attrs) = attrs.as_ref() else {
                     // no crate attributes, print out an error and exit
@@ -941,6 +955,46 @@ Available lint options:
     }
 }
 
+/// Show help for flag categories shared between rustdoc and rustc.
+///
+/// Returns whether a help option was printed.
+pub fn describe_flag_categories(matches: &Matches) -> bool {
+    // Handle the special case of -Wall.
+    let wall = matches.opt_strs("W");
+    if wall.iter().any(|x| *x == "all") {
+        print_wall_help();
+        rustc_errors::FatalError.raise();
+    }
+
+    // Don't handle -W help here, because we might first load plugins.
+    let debug_flags = matches.opt_strs("Z");
+    if debug_flags.iter().any(|x| *x == "help") {
+        describe_debug_flags();
+        return true;
+    }
+
+    let cg_flags = matches.opt_strs("C");
+    if cg_flags.iter().any(|x| *x == "help") {
+        describe_codegen_flags();
+        return true;
+    }
+
+    if cg_flags.iter().any(|x| *x == "no-stack-check") {
+        early_warn(
+            ErrorOutputType::default(),
+            "the --no-stack-check flag is deprecated and does nothing",
+        );
+    }
+
+    if cg_flags.iter().any(|x| *x == "passes=list") {
+        let backend_name = debug_flags.iter().find_map(|x| x.strip_prefix("codegen-backend="));
+        get_codegen_backend(&None, backend_name).print_passes();
+        return true;
+    }
+
+    false
+}
+
 fn describe_debug_flags() {
     println!("\nAvailable options:\n");
     print_flag_list("-Z", config::Z_OPTIONS);
@@ -951,7 +1005,7 @@ fn describe_codegen_flags() {
     print_flag_list("-C", config::CG_OPTIONS);
 }
 
-pub fn print_flag_list<T>(
+fn print_flag_list<T>(
     cmdline_opt: &str,
     flag_list: &[(&'static str, T, &'static str, &'static str)],
 ) {
@@ -1044,37 +1098,7 @@ pub fn handle_options(args: &[String]) -> Option<getopts::Matches> {
         return None;
     }
 
-    // Handle the special case of -Wall.
-    let wall = matches.opt_strs("W");
-    if wall.iter().any(|x| *x == "all") {
-        print_wall_help();
-        rustc_errors::FatalError.raise();
-    }
-
-    // Don't handle -W help here, because we might first load plugins.
-    let debug_flags = matches.opt_strs("Z");
-    if debug_flags.iter().any(|x| *x == "help") {
-        describe_debug_flags();
-        return None;
-    }
-
-    let cg_flags = matches.opt_strs("C");
-
-    if cg_flags.iter().any(|x| *x == "help") {
-        describe_codegen_flags();
-        return None;
-    }
-
-    if cg_flags.iter().any(|x| *x == "no-stack-check") {
-        early_warn(
-            ErrorOutputType::default(),
-            "the --no-stack-check flag is deprecated and does nothing",
-        );
-    }
-
-    if cg_flags.iter().any(|x| *x == "passes=list") {
-        let backend_name = debug_flags.iter().find_map(|x| x.strip_prefix("codegen-backend="));
-        get_codegen_backend(&None, backend_name).print_passes();
+    if describe_flag_categories(&matches) {
         return None;
     }
 
@@ -1245,11 +1269,9 @@ pub fn report_ice(info: &panic::PanicInfo<'_>, bug_report_url: &str) {
     interface::try_print_query_stack(&handler, num_frames);
 
     #[cfg(windows)]
-    unsafe {
-        if env::var("RUSTC_BREAK_ON_ICE").is_ok() {
-            // Trigger a debugger if we crashed during bootstrap
-            winapi::um::debugapi::DebugBreak();
-        }
+    if env::var("RUSTC_BREAK_ON_ICE").is_ok() {
+        // Trigger a debugger if we crashed during bootstrap
+        unsafe { windows::Win32::System::Diagnostics::Debug::DebugBreak() };
     }
 }
 
@@ -1355,9 +1377,9 @@ pub fn main() -> ! {
         RunCompiler::new(&args, &mut callbacks).run()
     });
 
-    if callbacks.time_passes {
+    if let Some(format) = callbacks.time_passes {
         let end_rss = get_resident_set_size();
-        print_time_passes_entry("total", start_time.elapsed(), start_rss, end_rss);
+        print_time_passes_entry("total", start_time.elapsed(), start_rss, end_rss, format);
     }
 
     process::exit(exit_code)
diff --git a/compiler/rustc_error_codes/src/error_codes.rs b/compiler/rustc_error_codes/src/error_codes.rs
index df857be85..d104ff089 100644
--- a/compiler/rustc_error_codes/src/error_codes.rs
+++ b/compiler/rustc_error_codes/src/error_codes.rs
@@ -513,6 +513,7 @@ E0790: include_str!("./error_codes/E0790.md"),
 E0791: include_str!("./error_codes/E0791.md"),
 E0792: include_str!("./error_codes/E0792.md"),
 E0793: include_str!("./error_codes/E0793.md"),
+E0794: include_str!("./error_codes/E0794.md"),
 }
 
 // Undocumented removed error codes. Note that many removed error codes are documented.
diff --git a/compiler/rustc_error_codes/src/error_codes/E0010.md b/compiler/rustc_error_codes/src/error_codes/E0010.md
index 71c790e10..f03f8a660 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0010.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0010.md
@@ -5,7 +5,5 @@ the heap at runtime, and therefore cannot be done at compile time.
 Erroneous code example:
 
 ```compile_fail,E0010
-#![feature(box_syntax)]
-
-const CON : Box<i32> = box 0;
+const CON : Vec<i32> = vec![1, 2, 3];
 ```
diff --git a/compiler/rustc_error_codes/src/error_codes/E0080.md b/compiler/rustc_error_codes/src/error_codes/E0080.md
index 7b1bbde61..c05324a5a 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0080.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0080.md
@@ -15,9 +15,9 @@ or causing an integer overflow are two ways to induce this error.
 
 Ensure that the expressions given can be evaluated as the desired integer type.
 
-See the [Custom Discriminants][custom-discriminants] section of the Reference
-for more information about setting custom integer types on fieldless enums
-using the [`repr` attribute][repr-attribute].
+See the [Discriminants] section of the Reference for more information about
+setting custom integer types on enums using the
+[`repr` attribute][repr-attribute].
 
-[custom-discriminants]: https://doc.rust-lang.org/reference/items/enumerations.html#custom-discriminant-values-for-field-less-enumerations
-[repr-attribute]: https://doc.rust-lang.org/reference/type-layout.html#reprc-enums
+[discriminants]: https://doc.rust-lang.org/reference/items/enumerations.html#discriminants
+[repr-attribute]: https://doc.rust-lang.org/reference/type-layout.html#representations
diff --git a/compiler/rustc_error_codes/src/error_codes/E0206.md b/compiler/rustc_error_codes/src/error_codes/E0206.md
index 4405a2149..9e85234bd 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0206.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0206.md
@@ -1,5 +1,5 @@
-The `Copy` trait was implemented on a type which is neither a struct nor an
-enum.
+The `Copy` trait was implemented on a type which is neither a struct, an
+enum, nor a union.
 
 Erroneous code example:
 
@@ -10,6 +10,6 @@ struct Bar;
 impl Copy for &'static mut Bar { } // error!
 ```
 
-You can only implement `Copy` for a struct or an enum.
+You can only implement `Copy` for a struct, an enum, or a union.
 The previous example will fail because `&'static mut Bar`
-is not a struct or enum.
+is not a struct, an enum, or a union.
diff --git a/compiler/rustc_error_codes/src/error_codes/E0223.md b/compiler/rustc_error_codes/src/error_codes/E0223.md
index 0d49f514c..ba5f00528 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0223.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0223.md
@@ -3,31 +3,33 @@ An attempt was made to retrieve an associated type, but the type was ambiguous.
 Erroneous code example:
 
 ```compile_fail,E0223
-trait MyTrait {type X; }
+trait Trait { type X; }
 
 fn main() {
-    let foo: MyTrait::X;
+    let foo: Trait::X;
 }
 ```
 
-The problem here is that we're attempting to take the type of X from MyTrait.
-Unfortunately, the type of X is not defined, because it's only made concrete in
-implementations of the trait. A working version of this code might look like:
+The problem here is that we're attempting to take the associated type of `X`
+from `Trait`. Unfortunately, the type of `X` is not defined, because it's only
+made concrete in implementations of the trait. A working version of this code
+might look like:
 
 ```
-trait MyTrait {type X; }
-struct MyStruct;
+trait Trait { type X; }
 
-impl MyTrait for MyStruct {
+struct Struct;
+impl Trait for Struct {
     type X = u32;
 }
 
 fn main() {
-    let foo: <MyStruct as MyTrait>::X;
+    let foo: <Struct as Trait>::X;
 }
 ```
 
-This syntax specifies that we want the X type from MyTrait, as made concrete in
-MyStruct. The reason that we cannot simply use `MyStruct::X` is that MyStruct
-might implement two different traits with identically-named associated types.
-This syntax allows disambiguation between the two.
+This syntax specifies that we want the associated type `X` from `Struct`'s
+implementation of `Trait`.
+
+Due to internal limitations of the current compiler implementation we cannot
+simply use `Struct::X`.
diff --git a/compiler/rustc_error_codes/src/error_codes/E0368.md b/compiler/rustc_error_codes/src/error_codes/E0368.md
index 7b9d93348..b18e8758d 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0368.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0368.md
@@ -41,7 +41,7 @@ impl Add for Foo {
 
 fn main() {
     let mut x: Foo = Foo(5);
-    x += Foo(7); // error, `+= cannot be applied to the type `Foo`
+    x += Foo(7); // error, `+=` cannot be applied to the type `Foo`
 }
 ```
 
diff --git a/compiler/rustc_error_codes/src/error_codes/E0416.md b/compiler/rustc_error_codes/src/error_codes/E0416.md
index 7bc316daf..8c0edcee5 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0416.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0416.md
@@ -23,6 +23,6 @@ Or maybe did you mean to unify? Consider using a guard:
 # let (A, B, C) = (1, 2, 3);
 match (A, B, C) {
     (x, x2, see) if x == x2 => { /* A and B are equal, do one thing */ }
-    (y, z, see) => { /* A and B unequal; do another thing */ }
+    (y, z, see) => { /* A and B not equal; do another thing */ }
 }
 ```
diff --git a/compiler/rustc_error_codes/src/error_codes/E0449.md b/compiler/rustc_error_codes/src/error_codes/E0449.md
index 9afc67689..a5876e075 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0449.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0449.md
@@ -1,4 +1,6 @@
-A visibility qualifier was used when it was unnecessary.
+A visibility qualifier was used where one is not permitted. Visibility
+qualifiers are not permitted on enum variants, trait items, impl blocks, and
+extern blocks, as they already share the visibility of the parent item.
 
 Erroneous code examples:
 
@@ -9,15 +11,18 @@ trait Foo {
     fn foo();
 }
 
-pub impl Bar {} // error: unnecessary visibility qualifier
+enum Baz {
+    pub Qux, // error: visibility qualifiers are not permitted here
+}
+
+pub impl Bar {} // error: visibility qualifiers are not permitted here
 
-pub impl Foo for Bar { // error: unnecessary visibility qualifier
-    pub fn foo() {} // error: unnecessary visibility qualifier
+pub impl Foo for Bar { // error: visibility qualifiers are not permitted here
+    pub fn foo() {} // error: visibility qualifiers are not permitted here
 }
 ```
 
-To fix this error, please remove the visibility qualifier when it is not
-required. Example:
+To fix this error, simply remove the visibility qualifier. Example:
 
 ```
 struct Bar;
@@ -26,12 +31,18 @@ trait Foo {
     fn foo();
 }
 
+enum Baz {
+    // Enum variants share the visibility of the enum they are in, so
+    // `pub` is not allowed here
+    Qux,
+}
+
 // Directly implemented methods share the visibility of the type itself,
-// so `pub` is unnecessary here
+// so `pub` is not allowed here
 impl Bar {}
 
-// Trait methods share the visibility of the trait, so `pub` is
-// unnecessary in either case
+// Trait methods share the visibility of the trait, so `pub` is not
+// allowed in either case
 impl Foo for Bar {
     fn foo() {}
 }
diff --git a/compiler/rustc_error_codes/src/error_codes/E0710.md b/compiler/rustc_error_codes/src/error_codes/E0710.md
index b7037ea61..84d55d524 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0710.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0710.md
@@ -3,14 +3,14 @@ An unknown tool name was found in a scoped lint.
 Erroneous code examples:
 
 ```compile_fail,E0710
-#[allow(clipp::filter_map)] // error!`
+#[allow(clipp::filter_map)] // error!
 fn main() {
     // business logic
 }
 ```
 
 ```compile_fail,E0710
-#[warn(clipp::filter_map)] // error!`
+#[warn(clipp::filter_map)] // error!
 fn main() {
     // business logic
 }
diff --git a/compiler/rustc_error_codes/src/error_codes/E0794.md b/compiler/rustc_error_codes/src/error_codes/E0794.md
new file mode 100644
index 000000000..c8f73de95
--- /dev/null
+++ b/compiler/rustc_error_codes/src/error_codes/E0794.md
@@ -0,0 +1,64 @@
+A lifetime parameter of a function definition is called *late-bound* if it both:
+
+1. appears in an argument type
+2. does not appear in a generic type constraint
+
+You cannot specify lifetime arguments for late-bound lifetime parameters.
+
+Erroneous code example:
+
+```compile_fail,E0794
+fn foo<'a>(x: &'a str) -> &'a str { x }
+let _ = foo::<'static>;
+```
+
+The type of a concrete instance of a generic function is universally quantified
+over late-bound lifetime parameters. This is because we want the function to
+work for any lifetime substituted for the late-bound lifetime parameter, no
+matter where the function is called. Consequently, it doesn't make sense to
+specify arguments for late-bound lifetime parameters, since they are not
+resolved until the function's call site(s).
+
+To fix the issue, remove the specified lifetime:
+
+```
+fn foo<'a>(x: &'a str) -> &'a str { x }
+let _ = foo;
+```
+
+### Additional information
+
+Lifetime parameters that are not late-bound are called *early-bound*.
+Confusion may arise from the fact that late-bound and early-bound
+lifetime parameters are declared the same way in function definitions.
+When referring to a function pointer type, universal quantification over
+late-bound lifetime parameters can be made explicit:
+
+```
+trait BarTrait<'a> {}
+
+struct Bar<'a> {
+    s: &'a str
+}
+
+impl<'a> BarTrait<'a> for Bar<'a> {}
+
+fn bar<'a, 'b, T>(x: &'a str, _t: T) -> &'a str
+where T: BarTrait<'b>
+{
+    x
+}
+
+let bar_fn: for<'a> fn(&'a str, Bar<'static>) -> &'a str = bar; // OK
+let bar_fn2 = bar::<'static, Bar>; // Not allowed
+let bar_fn3 = bar::<Bar>; // OK
+```
+
+In the definition of `bar`, the lifetime parameter `'a` is late-bound, while
+`'b` is early-bound. This is reflected in the type annotation for `bar_fn`,
+where `'a` is universally quantified and `'b` is substituted by a specific
+lifetime. It is not allowed to explicitly specify early-bound lifetime
+arguments when late-bound lifetime parameters are present (as for `bar_fn2`,
+see [issue #42868](https://github.com/rust-lang/rust/issues/42868)), although
+the types that are constrained by early-bound parameters can be specified (as
+for `bar_fn3`).
diff --git a/compiler/rustc_error_messages/locales/en-US.ftl b/compiler/rustc_error_messages/messages.ftl
index e62923744..e62923744 100644
--- a/compiler/rustc_error_messages/locales/en-US.ftl
+++ b/compiler/rustc_error_messages/messages.ftl
diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs
index 010e5f060..6f319b96f 100644
--- a/compiler/rustc_error_messages/src/lib.rs
+++ b/compiler/rustc_error_messages/src/lib.rs
@@ -1,5 +1,5 @@
 #![feature(let_chains)]
-#![feature(once_cell)]
+#![feature(lazy_cell)]
 #![feature(rustc_attrs)]
 #![feature(type_alias_impl_trait)]
 #![deny(rustc::untranslatable_diagnostic)]
@@ -34,7 +34,7 @@ use intl_memoizer::IntlLangMemoizer;
 pub use fluent_bundle::{self, types::FluentType, FluentArgs, FluentError, FluentValue};
 pub use unic_langid::{langid, LanguageIdentifier};
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub type FluentBundle = fluent_bundle::bundle::FluentBundle<FluentResource, IntlLangMemoizer>;
 
diff --git a/compiler/rustc_errors/Cargo.toml b/compiler/rustc_errors/Cargo.toml
index cadd53fbd..e1ead08ea 100644
--- a/compiler/rustc_errors/Cargo.toml
+++ b/compiler/rustc_errors/Cargo.toml
@@ -25,8 +25,14 @@ termize = "0.1.1"
 serde = { version = "1.0.125", features = [ "derive" ] }
 serde_json = "1.0.59"
 
-[target.'cfg(windows)'.dependencies]
-winapi = { version = "0.3", features = [ "handleapi", "synchapi", "winbase" ] }
+[target.'cfg(windows)'.dependencies.windows]
+version = "0.46.0"
+features = [
+    "Win32_Foundation",
+    "Win32_Security",
+    "Win32_System_Threading",
+    "Win32_System_WindowsProgramming",
+]
 
 [features]
 rustc_use_parallel_compiler = ['rustc_error_messages/rustc_use_parallel_compiler']
diff --git a/compiler/rustc_errors/locales/en-US.ftl b/compiler/rustc_errors/messages.ftl
index dde1d6c0a..dde1d6c0a 100644
--- a/compiler/rustc_errors/locales/en-US.ftl
+++ b/compiler/rustc_errors/messages.ftl
diff --git a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs
index d8879bf70..9872b3bda 100644
--- a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs
+++ b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs
@@ -202,7 +202,10 @@ impl AnnotateSnippetEmitterWriter {
                             annotations: annotations
                                 .iter()
                                 .map(|annotation| SourceAnnotation {
-                                    range: (annotation.start_col, annotation.end_col),
+                                    range: (
+                                        annotation.start_col.display,
+                                        annotation.end_col.display,
+                                    ),
                                     label: annotation.label.as_deref().unwrap_or_default(),
                                     annotation_type: annotation_type_for_level(*level),
                                 })
diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs
index 9ed8ab674..29c692128 100644
--- a/compiler/rustc_errors/src/diagnostic.rs
+++ b/compiler/rustc_errors/src/diagnostic.rs
@@ -880,6 +880,7 @@ impl Diagnostic {
     ///
     /// This is intended to be used for suggestions that are *very* obvious in what the changes
     /// need to be from the message, but we still want other tools to be able to apply them.
+    #[rustc_lint_diagnostics]
     pub fn tool_only_span_suggestion(
         &mut self,
         sp: Span,
@@ -956,7 +957,7 @@ impl Diagnostic {
     // Exact iteration order of diagnostic arguments shouldn't make a difference to output because
     // they're only used in interpolation.
     #[allow(rustc::potential_query_instability)]
-    pub fn args<'a>(&'a self) -> impl Iterator<Item = DiagnosticArg<'a, 'static>> {
+    pub fn args(&self) -> impl Iterator<Item = DiagnosticArg<'_, 'static>> {
         self.args.iter()
     }
 
diff --git a/compiler/rustc_errors/src/diagnostic_impls.rs b/compiler/rustc_errors/src/diagnostic_impls.rs
index e82bad67b..65f8a61a3 100644
--- a/compiler/rustc_errors/src/diagnostic_impls.rs
+++ b/compiler/rustc_errors/src/diagnostic_impls.rs
@@ -1,4 +1,4 @@
-use crate::fluent_generated as fluent;
+use crate::{fluent_generated as fluent, AddToDiagnostic};
 use crate::{DiagnosticArgValue, DiagnosticBuilder, Handler, IntoDiagnostic, IntoDiagnosticArg};
 use rustc_ast as ast;
 use rustc_ast_pretty::pprust;
@@ -6,6 +6,7 @@ use rustc_hir as hir;
 use rustc_lint_defs::Level;
 use rustc_span::edition::Edition;
 use rustc_span::symbol::{Ident, MacroRulesNormalizedIdent, Symbol};
+use rustc_span::Span;
 use rustc_target::abi::TargetDataLayoutErrors;
 use rustc_target::spec::{PanicStrategy, SplitDebuginfo, StackProtector, TargetTriple};
 use rustc_type_ir as type_ir;
@@ -276,3 +277,26 @@ impl IntoDiagnostic<'_, !> for TargetDataLayoutErrors<'_> {
         }
     }
 }
+
+/// Utility struct used to apply a single label while highlighting multiple spans
+pub struct SingleLabelManySpans {
+    pub spans: Vec<Span>,
+    pub label: &'static str,
+    pub kind: LabelKind,
+}
+impl AddToDiagnostic for SingleLabelManySpans {
+    fn add_to_diagnostic_with<F>(self, diag: &mut crate::Diagnostic, _: F) {
+        match self.kind {
+            LabelKind::Note => diag.span_note(self.spans, self.label),
+            LabelKind::Label => diag.span_labels(self.spans, self.label),
+            LabelKind::Help => diag.span_help(self.spans, self.label),
+        };
+    }
+}
+
+/// The kind of label to attach when using [`SingleLabelManySpans`]
+pub enum LabelKind {
+    Note,
+    Label,
+    Help,
+}
diff --git a/compiler/rustc_errors/src/emitter.rs b/compiler/rustc_errors/src/emitter.rs
index 211bbf4f5..fe44799ef 100644
--- a/compiler/rustc_errors/src/emitter.rs
+++ b/compiler/rustc_errors/src/emitter.rs
@@ -12,7 +12,9 @@ use Destination::*;
 use rustc_span::source_map::SourceMap;
 use rustc_span::{FileLines, SourceFile, Span};
 
-use crate::snippet::{Annotation, AnnotationType, Line, MultilineAnnotation, Style, StyledString};
+use crate::snippet::{
+    Annotation, AnnotationColumn, AnnotationType, Line, MultilineAnnotation, Style, StyledString,
+};
 use crate::styled_buffer::StyledBuffer;
 use crate::translation::{to_fluent_args, Translate};
 use crate::{
@@ -858,7 +860,7 @@ impl EmitterWriter {
         let mut short_start = true;
         for ann in &line.annotations {
             if let AnnotationType::MultilineStart(depth) = ann.annotation_type {
-                if source_string.chars().take(ann.start_col).all(|c| c.is_whitespace()) {
+                if source_string.chars().take(ann.start_col.display).all(|c| c.is_whitespace()) {
                     let style = if ann.is_primary {
                         Style::UnderlinePrimary
                     } else {
@@ -1093,15 +1095,15 @@ impl EmitterWriter {
                         '_',
                         line_offset + pos,
                         width_offset + depth,
-                        (code_offset + annotation.start_col).saturating_sub(left),
+                        (code_offset + annotation.start_col.display).saturating_sub(left),
                         style,
                     );
                 }
                 _ if self.teach => {
                     buffer.set_style_range(
                         line_offset,
-                        (code_offset + annotation.start_col).saturating_sub(left),
-                        (code_offset + annotation.end_col).saturating_sub(left),
+                        (code_offset + annotation.start_col.display).saturating_sub(left),
+                        (code_offset + annotation.end_col.display).saturating_sub(left),
                         style,
                         annotation.is_primary,
                     );
@@ -1133,7 +1135,7 @@ impl EmitterWriter {
                 for p in line_offset + 1..=line_offset + pos {
                     buffer.putc(
                         p,
-                        (code_offset + annotation.start_col).saturating_sub(left),
+                        (code_offset + annotation.start_col.display).saturating_sub(left),
                         '|',
                         style,
                     );
@@ -1169,9 +1171,9 @@ impl EmitterWriter {
             let style =
                 if annotation.is_primary { Style::LabelPrimary } else { Style::LabelSecondary };
             let (pos, col) = if pos == 0 {
-                (pos + 1, (annotation.end_col + 1).saturating_sub(left))
+                (pos + 1, (annotation.end_col.display + 1).saturating_sub(left))
             } else {
-                (pos + 2, annotation.start_col.saturating_sub(left))
+                (pos + 2, annotation.start_col.display.saturating_sub(left))
             };
             if let Some(ref label) = annotation.label {
                 buffer.puts(line_offset + pos, code_offset + col, label, style);
@@ -1208,7 +1210,7 @@ impl EmitterWriter {
             } else {
                 ('-', Style::UnderlineSecondary)
             };
-            for p in annotation.start_col..annotation.end_col {
+            for p in annotation.start_col.display..annotation.end_col.display {
                 buffer.putc(
                     line_offset + 1,
                     (code_offset + p).saturating_sub(left),
@@ -1405,7 +1407,7 @@ impl EmitterWriter {
                 // Account for newlines to align output to its label.
                 for (line, text) in normalize_whitespace(&text).lines().enumerate() {
                     buffer.append(
-                        0 + line,
+                        line,
                         &format!(
                             "{}{}",
                             if line == 0 { String::new() } else { " ".repeat(label_width) },
@@ -1459,7 +1461,7 @@ impl EmitterWriter {
                                         &annotated_file.file.name,
                                         line.line_index
                                     ),
-                                    annotations[0].start_col + 1,
+                                    annotations[0].start_col.file + 1,
                                 ),
                                 Style::LineAndColumn,
                             );
@@ -1546,7 +1548,7 @@ impl EmitterWriter {
                 buffer.prepend(buffer_msg_line_offset + 1, "::: ", Style::LineNumber);
                 let loc = if let Some(first_line) = annotated_file.lines.first() {
                     let col = if let Some(first_annotation) = first_line.annotations.first() {
-                        format!(":{}", first_annotation.start_col + 1)
+                        format!(":{}", first_annotation.start_col.file + 1)
                     } else {
                         String::new()
                     };
@@ -1607,8 +1609,8 @@ impl EmitterWriter {
                 let mut span_left_margin = usize::MAX;
                 for line in &annotated_file.lines {
                     for ann in &line.annotations {
-                        span_left_margin = min(span_left_margin, ann.start_col);
-                        span_left_margin = min(span_left_margin, ann.end_col);
+                        span_left_margin = min(span_left_margin, ann.start_col.display);
+                        span_left_margin = min(span_left_margin, ann.end_col.display);
                     }
                 }
                 if span_left_margin == usize::MAX {
@@ -1625,11 +1627,12 @@ impl EmitterWriter {
                         annotated_file.file.get_line(line.line_index - 1).map_or(0, |s| s.len()),
                     );
                     for ann in &line.annotations {
-                        span_right_margin = max(span_right_margin, ann.start_col);
-                        span_right_margin = max(span_right_margin, ann.end_col);
+                        span_right_margin = max(span_right_margin, ann.start_col.display);
+                        span_right_margin = max(span_right_margin, ann.end_col.display);
                         // FIXME: account for labels not in the same line
                         let label_right = ann.label.as_ref().map_or(0, |l| l.len() + 1);
-                        label_right_margin = max(label_right_margin, ann.end_col + label_right);
+                        label_right_margin =
+                            max(label_right_margin, ann.end_col.display + label_right);
                     }
                 }
 
@@ -1829,6 +1832,12 @@ impl EmitterWriter {
             }
             let show_code_change = if has_deletion && !is_multiline {
                 DisplaySuggestion::Diff
+            } else if let [part] = &parts[..]
+                && part.snippet.ends_with('\n')
+                && part.snippet.trim() == complete.trim()
+            {
+                // We are adding a line(s) of code before code that was already there.
+                DisplaySuggestion::Add
             } else if (parts.len() != 1 || parts[0].snippet.trim() != complete.trim())
                 && !is_multiline
             {
@@ -1876,7 +1885,10 @@ impl EmitterWriter {
                 row_num += line_end - line_start;
             }
             let mut unhighlighted_lines = Vec::new();
+            let mut last_pos = 0;
+            let mut is_item_attribute = false;
             for (line_pos, (line, highlight_parts)) in lines.by_ref().zip(highlights).enumerate() {
+                last_pos = line_pos;
                 debug!(%line_pos, %line, ?highlight_parts);
 
                 // Remember lines that are not highlighted to hide them if needed
@@ -1884,6 +1896,12 @@ impl EmitterWriter {
                     unhighlighted_lines.push((line_pos, line));
                     continue;
                 }
+                if highlight_parts.len() == 1
+                    && line.trim().starts_with("#[")
+                    && line.trim().ends_with(']')
+                {
+                    is_item_attribute = true;
+                }
 
                 match unhighlighted_lines.len() {
                     0 => (),
@@ -1895,7 +1913,7 @@ impl EmitterWriter {
                         self.draw_code_line(
                             &mut buffer,
                             &mut row_num,
-                            &Vec::new(),
+                            &[],
                             p + line_start,
                             l,
                             show_code_change,
@@ -1915,11 +1933,11 @@ impl EmitterWriter {
                         let last_line = unhighlighted_lines.pop();
                         let first_line = unhighlighted_lines.drain(..).next();
 
-                        first_line.map(|(p, l)| {
+                        if let Some((p, l)) = first_line {
                             self.draw_code_line(
                                 &mut buffer,
                                 &mut row_num,
-                                &Vec::new(),
+                                &[],
                                 p + line_start,
                                 l,
                                 show_code_change,
@@ -1927,16 +1945,16 @@ impl EmitterWriter {
                                 &file_lines,
                                 is_multiline,
                             )
-                        });
+                        }
 
                         buffer.puts(row_num, max_line_num_len - 1, "...", Style::LineNumber);
                         row_num += 1;
 
-                        last_line.map(|(p, l)| {
+                        if let Some((p, l)) = last_line {
                             self.draw_code_line(
                                 &mut buffer,
                                 &mut row_num,
-                                &Vec::new(),
+                                &[],
                                 p + line_start,
                                 l,
                                 show_code_change,
@@ -1944,14 +1962,14 @@ impl EmitterWriter {
                                 &file_lines,
                                 is_multiline,
                             )
-                        });
+                        }
                     }
                 }
 
                 self.draw_code_line(
                     &mut buffer,
                     &mut row_num,
-                    highlight_parts,
+                    &highlight_parts,
                     line_pos + line_start,
                     line,
                     show_code_change,
@@ -1960,13 +1978,41 @@ impl EmitterWriter {
                     is_multiline,
                 )
             }
+            if let DisplaySuggestion::Add = show_code_change && is_item_attribute {
+                // The suggestion adds an entire line of code, ending on a newline, so we'll also
+                // print the *following* line, to provide context of what we're advicing people to
+                // do. Otherwise you would only see contextless code that can be confused for
+                // already existing code, despite the colors and UI elements.
+                // We special case `#[derive(_)]\n` and other attribute suggestions, because those
+                // are the ones where context is most useful.
+                let file_lines = sm
+                    .span_to_lines(span.primary_span().unwrap().shrink_to_hi())
+                    .expect("span_to_lines failed when emitting suggestion");
+                let line_num = sm.lookup_char_pos(parts[0].span.lo()).line;
+                if let Some(line) = file_lines.file.get_line(line_num - 1) {
+                    let line = normalize_whitespace(&line);
+                    self.draw_code_line(
+                        &mut buffer,
+                        &mut row_num,
+                        &[],
+                        line_num + last_pos + 1,
+                        &line,
+                        DisplaySuggestion::None,
+                        max_line_num_len,
+                        &file_lines,
+                        is_multiline,
+                    )
+                }
+            }
 
             // This offset and the ones below need to be signed to account for replacement code
             // that is shorter than the original code.
             let mut offsets: Vec<(usize, isize)> = Vec::new();
             // Only show an underline in the suggestions if the suggestion is not the
             // entirety of the code being shown and the displayed code is not multiline.
-            if let DisplaySuggestion::Diff | DisplaySuggestion::Underline = show_code_change {
+            if let DisplaySuggestion::Diff | DisplaySuggestion::Underline | DisplaySuggestion::Add =
+                show_code_change
+            {
                 draw_col_separator_no_space(&mut buffer, row_num, max_line_num_len + 1);
                 for part in parts {
                     let span_start_pos = sm.lookup_char_pos(part.span.lo()).col_display;
@@ -2176,7 +2222,7 @@ impl EmitterWriter {
         &self,
         buffer: &mut StyledBuffer,
         row_num: &mut usize,
-        highlight_parts: &Vec<SubstitutionHighlight>,
+        highlight_parts: &[SubstitutionHighlight],
         line_num: usize,
         line_to_add: &str,
         show_code_change: DisplaySuggestion,
@@ -2232,7 +2278,7 @@ impl EmitterWriter {
             }
         } else if is_multiline {
             buffer.puts(*row_num, 0, &self.maybe_anonymized(line_num), Style::LineNumber);
-            match &highlight_parts[..] {
+            match &highlight_parts {
                 [SubstitutionHighlight { start: 0, end }] if *end == line_to_add.len() => {
                     buffer.puts(*row_num, max_line_num_len + 1, "+ ", Style::Addition);
                 }
@@ -2244,6 +2290,10 @@ impl EmitterWriter {
                 }
             }
             buffer.append(*row_num, &normalize_whitespace(line_to_add), Style::NoStyle);
+        } else if let DisplaySuggestion::Add = show_code_change {
+            buffer.puts(*row_num, 0, &self.maybe_anonymized(line_num), Style::LineNumber);
+            buffer.puts(*row_num, max_line_num_len + 1, "+ ", Style::Addition);
+            buffer.append(*row_num, &normalize_whitespace(line_to_add), Style::NoStyle);
         } else {
             buffer.puts(*row_num, 0, &self.maybe_anonymized(line_num), Style::LineNumber);
             draw_col_separator(buffer, *row_num, max_line_num_len + 1);
@@ -2278,6 +2328,7 @@ enum DisplaySuggestion {
     Underline,
     Diff,
     None,
+    Add,
 }
 
 impl FileWithAnnotatedLines {
@@ -2352,8 +2403,8 @@ impl FileWithAnnotatedLines {
                         depth: 1,
                         line_start: lo.line,
                         line_end: hi.line,
-                        start_col: lo.col_display,
-                        end_col: hi.col_display,
+                        start_col: AnnotationColumn::from_loc(&lo),
+                        end_col: AnnotationColumn::from_loc(&hi),
                         is_primary,
                         label,
                         overlaps_exactly: false,
@@ -2361,8 +2412,8 @@ impl FileWithAnnotatedLines {
                     multiline_annotations.push((lo.file, ml));
                 } else {
                     let ann = Annotation {
-                        start_col: lo.col_display,
-                        end_col: hi.col_display,
+                        start_col: AnnotationColumn::from_loc(&lo),
+                        end_col: AnnotationColumn::from_loc(&hi),
                         is_primary,
                         label,
                         annotation_type: AnnotationType::Singleline,
@@ -2551,7 +2602,13 @@ fn num_overlap(
     (b_start..b_end + extra).contains(&a_start) || (a_start..a_end + extra).contains(&b_start)
 }
 fn overlaps(a1: &Annotation, a2: &Annotation, padding: usize) -> bool {
-    num_overlap(a1.start_col, a1.end_col + padding, a2.start_col, a2.end_col, false)
+    num_overlap(
+        a1.start_col.display,
+        a1.end_col.display + padding,
+        a2.start_col.display,
+        a2.end_col.display,
+        false,
+    )
 }
 
 fn emit_to_destination(
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index 09bf28ed4..d20b16890 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -6,7 +6,6 @@
 #![feature(array_windows)]
 #![feature(drain_filter)]
 #![feature(if_let_guard)]
-#![feature(is_terminal)]
 #![feature(adt_const_params)]
 #![feature(let_chains)]
 #![feature(never_type)]
@@ -76,7 +75,7 @@ pub use snippet::Style;
 pub type PErr<'a> = DiagnosticBuilder<'a, ErrorGuaranteed>;
 pub type PResult<'a, T> = Result<T, PErr<'a>>;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 // `PResult` is used a lot. Make sure it doesn't unintentionally get bigger.
 // (See also the comment on `DiagnosticBuilderInner`'s `diagnostic` field.)
@@ -331,7 +330,7 @@ impl CodeSuggestion {
                     });
                     buf.push_str(&part.snippet);
                     let cur_hi = sm.lookup_char_pos(part.span.hi());
-                    if prev_hi.line == cur_lo.line && cur_hi.line == cur_lo.line {
+                    if cur_hi.line == cur_lo.line && !part.snippet.is_empty() {
                         // Account for the difference between the width of the current code and the
                         // snippet being suggested, so that the *later* suggestions are correctly
                         // aligned on the screen.
@@ -383,7 +382,9 @@ pub use diagnostic::{
     DiagnosticStyledString, IntoDiagnosticArg, SubDiagnostic,
 };
 pub use diagnostic_builder::{DiagnosticBuilder, EmissionGuarantee, Noted};
-pub use diagnostic_impls::{DiagnosticArgFromDisplay, DiagnosticSymbolList};
+pub use diagnostic_impls::{
+    DiagnosticArgFromDisplay, DiagnosticSymbolList, LabelKind, SingleLabelManySpans,
+};
 use std::backtrace::Backtrace;
 
 /// A handler deals with errors and other compiler output.
diff --git a/compiler/rustc_errors/src/lock.rs b/compiler/rustc_errors/src/lock.rs
index a73472021..7db262abf 100644
--- a/compiler/rustc_errors/src/lock.rs
+++ b/compiler/rustc_errors/src/lock.rs
@@ -16,10 +16,12 @@ pub fn acquire_global_lock(name: &str) -> Box<dyn Any> {
     use std::ffi::CString;
     use std::io;
 
-    use winapi::shared::ntdef::HANDLE;
-    use winapi::um::handleapi::CloseHandle;
-    use winapi::um::synchapi::{CreateMutexA, ReleaseMutex, WaitForSingleObject};
-    use winapi::um::winbase::{INFINITE, WAIT_ABANDONED, WAIT_OBJECT_0};
+    use windows::{
+        core::PCSTR,
+        Win32::Foundation::{CloseHandle, HANDLE, WAIT_ABANDONED, WAIT_OBJECT_0},
+        Win32::System::Threading::{CreateMutexA, ReleaseMutex, WaitForSingleObject},
+        Win32::System::WindowsProgramming::INFINITE,
+    };
 
     struct Handle(HANDLE);
 
@@ -42,49 +44,38 @@ pub fn acquire_global_lock(name: &str) -> Box<dyn Any> {
     }
 
     let cname = CString::new(name).unwrap();
-    unsafe {
-        // Create a named mutex, with no security attributes and also not
-        // acquired when we create it.
-        //
-        // This will silently create one if it doesn't already exist, or it'll
-        // open up a handle to one if it already exists.
-        let mutex = CreateMutexA(std::ptr::null_mut(), 0, cname.as_ptr());
-        if mutex.is_null() {
-            panic!(
-                "failed to create global mutex named `{}`: {}",
-                name,
-                io::Error::last_os_error()
-            );
-        }
-        let mutex = Handle(mutex);
+    // Create a named mutex, with no security attributes and also not
+    // acquired when we create it.
+    //
+    // This will silently create one if it doesn't already exist, or it'll
+    // open up a handle to one if it already exists.
+    let mutex = unsafe { CreateMutexA(None, false, PCSTR::from_raw(cname.as_ptr().cast())) }
+        .unwrap_or_else(|_| panic!("failed to create global mutex named `{}`", name));
+    let mutex = Handle(mutex);
 
-        // Acquire the lock through `WaitForSingleObject`.
-        //
-        // A return value of `WAIT_OBJECT_0` means we successfully acquired it.
-        //
-        // A return value of `WAIT_ABANDONED` means that the previous holder of
-        // the thread exited without calling `ReleaseMutex`. This can happen,
-        // for example, when the compiler crashes or is interrupted via ctrl-c
-        // or the like. In this case, however, we are still transferred
-        // ownership of the lock so we continue.
-        //
-        // If an error happens.. well... that's surprising!
-        match WaitForSingleObject(mutex.0, INFINITE) {
-            WAIT_OBJECT_0 | WAIT_ABANDONED => {}
-            code => {
-                panic!(
-                    "WaitForSingleObject failed on global mutex named \
-                        `{}`: {} (ret={:x})",
-                    name,
-                    io::Error::last_os_error(),
-                    code
-                );
-            }
-        }
-
-        // Return a guard which will call `ReleaseMutex` when dropped.
-        Box::new(Guard(mutex))
+    // Acquire the lock through `WaitForSingleObject`.
+    //
+    // A return value of `WAIT_OBJECT_0` means we successfully acquired it.
+    //
+    // A return value of `WAIT_ABANDONED` means that the previous holder of
+    // the thread exited without calling `ReleaseMutex`. This can happen,
+    // for example, when the compiler crashes or is interrupted via ctrl-c
+    // or the like. In this case, however, we are still transferred
+    // ownership of the lock so we continue.
+    //
+    // If an error happens.. well... that's surprising!
+    match unsafe { WaitForSingleObject(mutex.0, INFINITE) } {
+        WAIT_OBJECT_0 | WAIT_ABANDONED => (),
+        err => panic!(
+            "WaitForSingleObject failed on global mutex named `{}`: {} (ret={:x})",
+            name,
+            io::Error::last_os_error(),
+            err.0
+        ),
     }
+
+    // Return a guard which will call `ReleaseMutex` when dropped.
+    Box::new(Guard(mutex))
 }
 
 #[cfg(not(windows))]
diff --git a/compiler/rustc_errors/src/snippet.rs b/compiler/rustc_errors/src/snippet.rs
index e4cc44c41..98eb70b5f 100644
--- a/compiler/rustc_errors/src/snippet.rs
+++ b/compiler/rustc_errors/src/snippet.rs
@@ -1,6 +1,6 @@
 // Code for annotating snippets.
 
-use crate::Level;
+use crate::{Level, Loc};
 
 #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq)]
 pub struct Line {
@@ -8,13 +8,39 @@ pub struct Line {
     pub annotations: Vec<Annotation>,
 }
 
+#[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Default)]
+pub struct AnnotationColumn {
+    /// the (0-indexed) column for *display* purposes, counted in characters, not utf-8 bytes
+    pub display: usize,
+    /// the (0-indexed) column in the file, counted in characters, not utf-8 bytes.
+    ///
+    /// this may be different from `self.display`,
+    /// e.g. if the file contains hard tabs, because we convert tabs to spaces for error messages.
+    ///
+    /// for example:
+    /// ```text
+    /// (hard tab)hello
+    ///           ^ this is display column 4, but file column 1
+    /// ```
+    ///
+    /// we want to keep around the correct file offset so that column numbers in error messages
+    /// are correct. (motivated by <https://github.com/rust-lang/rust/issues/109537>)
+    pub file: usize,
+}
+
+impl AnnotationColumn {
+    pub fn from_loc(loc: &Loc) -> AnnotationColumn {
+        AnnotationColumn { display: loc.col_display, file: loc.col.0 }
+    }
+}
+
 #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq)]
 pub struct MultilineAnnotation {
     pub depth: usize,
     pub line_start: usize,
     pub line_end: usize,
-    pub start_col: usize,
-    pub end_col: usize,
+    pub start_col: AnnotationColumn,
+    pub end_col: AnnotationColumn,
     pub is_primary: bool,
     pub label: Option<String>,
     pub overlaps_exactly: bool,
@@ -36,7 +62,12 @@ impl MultilineAnnotation {
     pub fn as_start(&self) -> Annotation {
         Annotation {
             start_col: self.start_col,
-            end_col: self.start_col + 1,
+            end_col: AnnotationColumn {
+                // these might not correspond to the same place anymore,
+                // but that's okay for our purposes
+                display: self.start_col.display + 1,
+                file: self.start_col.file + 1,
+            },
             is_primary: self.is_primary,
             label: None,
             annotation_type: AnnotationType::MultilineStart(self.depth),
@@ -45,7 +76,12 @@ impl MultilineAnnotation {
 
     pub fn as_end(&self) -> Annotation {
         Annotation {
-            start_col: self.end_col.saturating_sub(1),
+            start_col: AnnotationColumn {
+                // these might not correspond to the same place anymore,
+                // but that's okay for our purposes
+                display: self.end_col.display.saturating_sub(1),
+                file: self.end_col.file.saturating_sub(1),
+            },
             end_col: self.end_col,
             is_primary: self.is_primary,
             label: self.label.clone(),
@@ -55,8 +91,8 @@ impl MultilineAnnotation {
 
     pub fn as_line(&self) -> Annotation {
         Annotation {
-            start_col: 0,
-            end_col: 0,
+            start_col: Default::default(),
+            end_col: Default::default(),
             is_primary: self.is_primary,
             label: None,
             annotation_type: AnnotationType::MultilineLine(self.depth),
@@ -92,14 +128,14 @@ pub enum AnnotationType {
 
 #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq)]
 pub struct Annotation {
-    /// Start column, 0-based indexing -- counting *characters*, not
-    /// utf-8 bytes. Note that it is important that this field goes
+    /// Start column.
+    /// Note that it is important that this field goes
     /// first, so that when we sort, we sort orderings by start
     /// column.
-    pub start_col: usize,
+    pub start_col: AnnotationColumn,
 
     /// End column within the line (exclusive)
-    pub end_col: usize,
+    pub end_col: AnnotationColumn,
 
     /// Is this annotation derived from primary span
     pub is_primary: bool,
@@ -118,12 +154,13 @@ impl Annotation {
         matches!(self.annotation_type, AnnotationType::MultilineLine(_))
     }
 
+    /// Length of this annotation as displayed in the stderr output
     pub fn len(&self) -> usize {
         // Account for usize underflows
-        if self.end_col > self.start_col {
-            self.end_col - self.start_col
+        if self.end_col.display > self.start_col.display {
+            self.end_col.display - self.start_col.display
         } else {
-            self.start_col - self.end_col
+            self.start_col.display - self.end_col.display
         }
     }
 
diff --git a/compiler/rustc_expand/locales/en-US.ftl b/compiler/rustc_expand/messages.ftl
index cfae781bd..5d999d0db 100644
--- a/compiler/rustc_expand/locales/en-US.ftl
+++ b/compiler/rustc_expand/messages.ftl
@@ -135,4 +135,4 @@ expand_proc_macro_panicked =
     .help = message: {$message}
 
 expand_proc_macro_derive_tokens =
-    proc-macro derive produced unparseable tokens
+    proc-macro derive produced unparsable tokens
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index 22bc90f5c..caa2a201c 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -12,13 +12,13 @@ use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{AssocCtxt, Visitor};
 use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind};
 use rustc_attr::{self as attr, Deprecation, Stability};
-use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{
     Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, MultiSpan, PResult,
 };
 use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
-use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiagnostics};
+use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiagnostics, RegisteredTools};
 use rustc_parse::{self, parser, MACRO_ARGUMENTS};
 use rustc_session::errors::report_lit_error;
 use rustc_session::{parse::ParseSess, Limit, Session};
@@ -776,16 +776,14 @@ impl SyntaxExtension {
         let allow_internal_unstable =
             attr::allow_internal_unstable(sess, &attrs).collect::<Vec<Symbol>>();
 
-        let allow_internal_unsafe = sess.contains_name(attrs, sym::allow_internal_unsafe);
-        let local_inner_macros = sess
-            .find_by_name(attrs, sym::macro_export)
+        let allow_internal_unsafe = attr::contains_name(attrs, sym::allow_internal_unsafe);
+        let local_inner_macros = attr::find_by_name(attrs, sym::macro_export)
             .and_then(|macro_export| macro_export.meta_item_list())
             .map_or(false, |l| attr::list_contains_name(&l, sym::local_inner_macros));
-        let collapse_debuginfo = sess.contains_name(attrs, sym::collapse_debuginfo);
+        let collapse_debuginfo = attr::contains_name(attrs, sym::collapse_debuginfo);
         tracing::debug!(?local_inner_macros, ?collapse_debuginfo, ?allow_internal_unsafe);
 
-        let (builtin_name, helper_attrs) = sess
-            .find_by_name(attrs, sym::rustc_builtin_macro)
+        let (builtin_name, helper_attrs) = attr::find_by_name(attrs, sym::rustc_builtin_macro)
             .map(|attr| {
                 // Override `helper_attrs` passed above if it's a built-in macro,
                 // marking `proc_macro_derive` macros as built-in is not a realistic use case.
@@ -795,7 +793,9 @@ impl SyntaxExtension {
                 )
             })
             .unwrap_or_else(|| (None, helper_attrs));
-        let (stability, const_stability, body_stability) = attr::find_stability(&sess, attrs, span);
+        let stability = attr::find_stability(&sess, attrs, span);
+        let const_stability = attr::find_const_stability(&sess, attrs, span);
+        let body_stability = attr::find_body_stability(&sess, attrs);
         if let Some((_, sp)) = const_stability {
             sess.emit_err(errors::MacroConstStability {
                 span: sp,
@@ -947,14 +947,14 @@ pub trait ResolverExpand {
     fn declare_proc_macro(&mut self, id: NodeId);
 
     /// Tools registered with `#![register_tool]` and used by tool attributes and lints.
-    fn registered_tools(&self) -> &FxHashSet<Ident>;
+    fn registered_tools(&self) -> &RegisteredTools;
 }
 
 pub trait LintStoreExpand {
     fn pre_expansion_lint(
         &self,
         sess: &Session,
-        registered_tools: &FxHashSet<Ident>,
+        registered_tools: &RegisteredTools,
         node_id: NodeId,
         attrs: &[Attribute],
         items: &[P<Item>],
@@ -1004,6 +1004,7 @@ pub struct ExpansionData {
 pub struct ExtCtxt<'a> {
     pub sess: &'a Session,
     pub ecfg: expand::ExpansionConfig<'a>,
+    pub num_standard_library_imports: usize,
     pub reduced_recursion_limit: Option<Limit>,
     pub root_path: PathBuf,
     pub resolver: &'a mut dyn ResolverExpand,
@@ -1032,6 +1033,7 @@ impl<'a> ExtCtxt<'a> {
         ExtCtxt {
             sess,
             ecfg,
+            num_standard_library_imports: 0,
             reduced_recursion_limit: None,
             resolver,
             lint_store,
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
index 8a78c3296..264f30fb1 100644
--- a/compiler/rustc_expand/src/build.rs
+++ b/compiler/rustc_expand/src/build.rs
@@ -36,7 +36,7 @@ impl<'a> ExtCtxt<'a> {
         );
         let args = if !args.is_empty() {
             let args = args.into_iter().map(ast::AngleBracketedArg::Arg).collect();
-            ast::AngleBracketedArgs { args, span }.into()
+            Some(ast::AngleBracketedArgs { args, span }.into())
         } else {
             None
         };
@@ -620,10 +620,15 @@ impl<'a> ExtCtxt<'a> {
         span: Span,
         name: Ident,
         ty: P<ast::Ty>,
-        mutbl: ast::Mutability,
+        mutability: ast::Mutability,
         expr: P<ast::Expr>,
     ) -> P<ast::Item> {
-        self.item(span, name, AttrVec::new(), ast::ItemKind::Static(ty, mutbl, Some(expr)))
+        self.item(
+            span,
+            name,
+            AttrVec::new(),
+            ast::ItemKind::Static(ast::StaticItem { ty, mutability, expr: Some(expr) }.into()),
+        )
     }
 
     pub fn item_const(
@@ -633,8 +638,13 @@ impl<'a> ExtCtxt<'a> {
         ty: P<ast::Ty>,
         expr: P<ast::Expr>,
     ) -> P<ast::Item> {
-        let def = ast::Defaultness::Final;
-        self.item(span, name, AttrVec::new(), ast::ItemKind::Const(def, ty, Some(expr)))
+        let defaultness = ast::Defaultness::Final;
+        self.item(
+            span,
+            name,
+            AttrVec::new(),
+            ast::ItemKind::Const(ast::ConstItem { defaultness, ty, expr: Some(expr) }.into()),
+        )
     }
 
     // Builds `#[name]`.
diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs
index 01500c2c7..4ff8e409d 100644
--- a/compiler/rustc_expand/src/config.rs
+++ b/compiler/rustc_expand/src/config.rs
@@ -12,8 +12,8 @@ use rustc_ast::tokenstream::{LazyAttrTokenStream, TokenTree};
 use rustc_ast::NodeId;
 use rustc_ast::{self as ast, AttrStyle, Attribute, HasAttrs, HasTokens, MetaItem};
 use rustc_attr as attr;
+use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::map_in_place::MapInPlace;
 use rustc_feature::{Feature, Features, State as FeatureState};
 use rustc_feature::{
     ACCEPTED_FEATURES, ACTIVE_FEATURES, REMOVED_FEATURES, STABLE_REMOVED_FEATURES,
@@ -24,7 +24,6 @@ use rustc_session::Session;
 use rustc_span::edition::{Edition, ALL_EDITIONS};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
-use thin_vec::ThinVec;
 
 /// A folder that strips out items that do not belong in the current configuration.
 pub struct StripUnconfigured<'a> {
@@ -37,7 +36,7 @@ pub struct StripUnconfigured<'a> {
     pub lint_node_id: NodeId,
 }
 
-fn get_features(sess: &Session, krate_attrs: &[ast::Attribute]) -> Features {
+pub fn features(sess: &Session, krate_attrs: &[Attribute]) -> Features {
     fn feature_removed(sess: &Session, span: Span, reason: Option<&str>) {
         sess.emit_err(FeatureRemoved {
             span,
@@ -191,39 +190,16 @@ fn get_features(sess: &Session, krate_attrs: &[ast::Attribute]) -> Features {
     features
 }
 
-/// `cfg_attr`-process the crate's attributes and compute the crate's features.
-pub fn features(
-    sess: &Session,
-    mut krate: ast::Crate,
-    lint_node_id: NodeId,
-) -> (ast::Crate, Features) {
-    let mut strip_unconfigured =
-        StripUnconfigured { sess, features: None, config_tokens: false, lint_node_id };
-
-    let unconfigured_attrs = krate.attrs.clone();
-    let diag = &sess.parse_sess.span_diagnostic;
-    let err_count = diag.err_count();
-    let features = match strip_unconfigured.configure_krate_attrs(krate.attrs) {
-        None => {
-            // The entire crate is unconfigured.
-            krate.attrs = ast::AttrVec::new();
-            krate.items = ThinVec::new();
-            Features::default()
-        }
-        Some(attrs) => {
-            krate.attrs = attrs;
-            let features = get_features(sess, &krate.attrs);
-            if err_count == diag.err_count() {
-                // Avoid reconfiguring malformed `cfg_attr`s.
-                strip_unconfigured.features = Some(&features);
-                // Run configuration again, this time with features available
-                // so that we can perform feature-gating.
-                strip_unconfigured.configure_krate_attrs(unconfigured_attrs);
-            }
-            features
-        }
+pub fn pre_configure_attrs(sess: &Session, attrs: &[Attribute]) -> ast::AttrVec {
+    let strip_unconfigured = StripUnconfigured {
+        sess,
+        features: None,
+        config_tokens: false,
+        lint_node_id: ast::CRATE_NODE_ID,
     };
-    (krate, features)
+    let attrs: ast::AttrVec =
+        attrs.iter().flat_map(|attr| strip_unconfigured.process_cfg_attr(attr)).collect();
+    if strip_unconfigured.in_cfg(&attrs) { attrs } else { ast::AttrVec::new() }
 }
 
 #[macro_export]
@@ -254,11 +230,6 @@ impl<'a> StripUnconfigured<'a> {
         }
     }
 
-    fn configure_krate_attrs(&self, mut attrs: ast::AttrVec) -> Option<ast::AttrVec> {
-        attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
-        self.in_cfg(&attrs).then_some(attrs)
-    }
-
     /// Performs cfg-expansion on `stream`, producing a new `AttrTokenStream`.
     /// This is only used during the invocation of `derive` proc-macros,
     /// which require that we cfg-expand their entire input.
@@ -281,7 +252,7 @@ impl<'a> StripUnconfigured<'a> {
             .iter()
             .flat_map(|tree| match tree.clone() {
                 AttrTokenTree::Attributes(mut data) => {
-                    data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
+                    data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(&attr));
 
                     if self.in_cfg(&data.attrs) {
                         data.tokens = LazyAttrTokenStream::new(
@@ -319,12 +290,16 @@ impl<'a> StripUnconfigured<'a> {
     /// the syntax of any `cfg_attr` is incorrect.
     fn process_cfg_attrs<T: HasAttrs>(&self, node: &mut T) {
         node.visit_attrs(|attrs| {
-            attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
+            attrs.flat_map_in_place(|attr| self.process_cfg_attr(&attr));
         });
     }
 
-    fn process_cfg_attr(&self, attr: Attribute) -> Vec<Attribute> {
-        if attr.has_name(sym::cfg_attr) { self.expand_cfg_attr(attr, true) } else { vec![attr] }
+    fn process_cfg_attr(&self, attr: &Attribute) -> Vec<Attribute> {
+        if attr.has_name(sym::cfg_attr) {
+            self.expand_cfg_attr(attr, true)
+        } else {
+            vec![attr.clone()]
+        }
     }
 
     /// Parse and expand a single `cfg_attr` attribute into a list of attributes
@@ -334,9 +309,9 @@ impl<'a> StripUnconfigured<'a> {
     /// Gives a compiler warning when the `cfg_attr` contains no attributes and
     /// is in the original source file. Gives a compiler error if the syntax of
     /// the attribute is incorrect.
-    pub(crate) fn expand_cfg_attr(&self, attr: Attribute, recursive: bool) -> Vec<Attribute> {
+    pub(crate) fn expand_cfg_attr(&self, attr: &Attribute, recursive: bool) -> Vec<Attribute> {
         let Some((cfg_predicate, expanded_attrs)) =
-            rustc_parse::parse_cfg_attr(&attr, &self.sess.parse_sess) else {
+            rustc_parse::parse_cfg_attr(attr, &self.sess.parse_sess) else {
                 return vec![];
             };
 
@@ -365,10 +340,10 @@ impl<'a> StripUnconfigured<'a> {
             //  `#[cfg_attr(false, cfg_attr(true, some_attr))]`.
             expanded_attrs
                 .into_iter()
-                .flat_map(|item| self.process_cfg_attr(self.expand_cfg_attr_item(&attr, item)))
+                .flat_map(|item| self.process_cfg_attr(&self.expand_cfg_attr_item(attr, item)))
                 .collect()
         } else {
-            expanded_attrs.into_iter().map(|item| self.expand_cfg_attr_item(&attr, item)).collect()
+            expanded_attrs.into_iter().map(|item| self.expand_cfg_attr_item(attr, item)).collect()
         }
     }
 
@@ -491,7 +466,7 @@ impl<'a> StripUnconfigured<'a> {
         //
         // N.B., this is intentionally not part of the visit_expr() function
         //     in order for filter_map_expr() to be able to avoid this check
-        if let Some(attr) = expr.attrs().iter().find(|a| is_cfg(*a)) {
+        if let Some(attr) = expr.attrs().iter().find(|a| is_cfg(a)) {
             self.sess.emit_err(RemoveExprNotSupported { span: attr.span });
         }
 
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index 79d058d9c..ec4091154 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -20,7 +20,7 @@ use rustc_ast::{ForeignItemKind, HasAttrs, HasNodeId};
 use rustc_ast::{Inline, ItemKind, MacStmtStyle, MetaItemKind, ModKind};
 use rustc_ast::{NestedMetaItem, NodeId, PatKind, StmtKind, TyKind};
 use rustc_ast_pretty::pprust;
-use rustc_data_structures::map_in_place::MapInPlace;
+use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::PResult;
 use rustc_feature::Features;
@@ -1038,6 +1038,9 @@ trait InvocationCollectorNode: HasAttrs + HasNodeId + Sized {
     ) -> Result<Self::OutputTy, Self> {
         Ok(noop_flat_map(node, collector))
     }
+    fn expand_cfg_false(&mut self, collector: &mut InvocationCollector<'_, '_>, span: Span) {
+        collector.cx.emit_err(RemoveNodeNotSupported { span, descr: Self::descr() });
+    }
 }
 
 impl InvocationCollectorNode for P<ast::Item> {
@@ -1378,6 +1381,11 @@ impl InvocationCollectorNode for ast::Crate {
     fn noop_visit<V: MutVisitor>(&mut self, visitor: &mut V) {
         noop_visit_crate(self, visitor)
     }
+    fn expand_cfg_false(&mut self, collector: &mut InvocationCollector<'_, '_>, _span: Span) {
+        self.attrs.clear();
+        // Standard prelude imports are left in the crate for backward compatibility.
+        self.items.truncate(collector.cx.num_standard_library_imports);
+    }
 }
 
 impl InvocationCollectorNode for P<ast::Ty> {
@@ -1688,7 +1696,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
         res
     }
 
-    fn expand_cfg_attr(&self, node: &mut impl HasAttrs, attr: ast::Attribute, pos: usize) {
+    fn expand_cfg_attr(&self, node: &mut impl HasAttrs, attr: &ast::Attribute, pos: usize) {
         node.visit_attrs(|attrs| {
             // Repeated `insert` calls is inefficient, but the number of
             // insertions is almost always 0 or 1 in practice.
@@ -1712,7 +1720,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                         Default::default()
                     }
                     sym::cfg_attr => {
-                        self.expand_cfg_attr(&mut node, attr, pos);
+                        self.expand_cfg_attr(&mut node, &attr, pos);
                         continue;
                     }
                     _ => {
@@ -1756,11 +1764,11 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                             continue;
                         }
 
-                        self.cx.emit_err(RemoveNodeNotSupported { span, descr: Node::descr() });
+                        node.expand_cfg_false(self, span);
                         continue;
                     }
                     sym::cfg_attr => {
-                        self.expand_cfg_attr(node, attr, pos);
+                        self.expand_cfg_attr(node, &attr, pos);
                         continue;
                     }
                     _ => visit_clobber(node, |node| {
diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs
index 634e206e5..ced7531c3 100644
--- a/compiler/rustc_expand/src/lib.rs
+++ b/compiler/rustc_expand/src/lib.rs
@@ -64,4 +64,4 @@ mod mut_visit {
     mod tests;
 }
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs
index f469b2dae..355722922 100644
--- a/compiler/rustc_expand/src/mbe/diagnostics.rs
+++ b/compiler/rustc_expand/src/mbe/diagnostics.rs
@@ -1,12 +1,10 @@
-use std::borrow::Cow;
-
 use crate::base::{DummyResult, ExtCtxt, MacResult};
 use crate::expand::{parse_ast_fragment, AstFragmentKind};
 use crate::mbe::{
     macro_parser::{MatcherLoc, NamedParseResult, ParseResult::*, TtParser},
     macro_rules::{try_match_macro, Tracker},
 };
-use rustc_ast::token::{self, Token};
+use rustc_ast::token::{self, Token, TokenKind};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast_pretty::pprust;
 use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, DiagnosticMessage};
@@ -14,6 +12,7 @@ use rustc_parse::parser::{Parser, Recovery};
 use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
+use std::borrow::Cow;
 
 use super::macro_rules::{parser_from_cx, NoopTracker};
 
@@ -63,6 +62,18 @@ pub(super) fn failed_to_match_macro<'cx>(
         err.note(format!("while trying to match {remaining_matcher}"));
     }
 
+    if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
+        && (matches!(expected_token.kind, TokenKind::Interpolated(_))
+            || matches!(token.kind, TokenKind::Interpolated(_)))
+    {
+        err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens");
+        err.note("see <https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment> for more information");
+
+        if !def_span.is_dummy() && !cx.source_map().is_imported(def_span) {
+            err.help("try using `:tt` instead in the macro definition");
+        }
+    }
+
     // Check whether there's a missing comma in this macro call, like `println!("{}" a);`
     if let Some((arg, comma_span)) = arg.add_comma() {
         for lhs in lhses {
@@ -239,12 +250,24 @@ pub(super) fn emit_frag_parse_err(
                 e.note(
                     "the macro call doesn't expand to an expression, but it can expand to a statement",
                 );
-                e.span_suggestion_verbose(
-                    site_span.shrink_to_hi(),
-                    "add `;` to interpret the expansion as a statement",
-                    ";",
-                    Applicability::MaybeIncorrect,
-                );
+
+                if parser.token == token::Semi {
+                    if let Ok(snippet) = parser.sess.source_map().span_to_snippet(site_span) {
+                        e.span_suggestion_verbose(
+                            site_span,
+                            "surround the macro invocation with `{}` to interpret the expansion as a statement",
+                            format!("{{ {}; }}", snippet),
+                            Applicability::MaybeIncorrect,
+                        );
+                    }
+                } else {
+                    e.span_suggestion_verbose(
+                        site_span.shrink_to_hi(),
+                        "add `;` to interpret the expansion as a statement",
+                        ";",
+                        Applicability::MaybeIncorrect,
+                    );
+                }
             }
         },
         _ => annotate_err_with_kind(&mut e, kind, site_span),
diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs
index de34df011..fb3a00d86 100644
--- a/compiler/rustc_expand/src/mbe/metavar_expr.rs
+++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs
@@ -41,7 +41,7 @@ impl MetaVarExpr {
         };
         check_trailing_token(&mut tts, sess)?;
         let mut iter = args.trees();
-        let rslt = match &*ident.as_str() {
+        let rslt = match ident.as_str() {
             "count" => parse_count(&mut iter, sess, ident.span)?,
             "ignore" => MetaVarExpr::Ignore(parse_ident(&mut iter, sess, ident.span)?),
             "index" => MetaVarExpr::Index(parse_depth(&mut iter, sess, ident.span)?),
diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs
index 47a8b4bc4..a07cb6517 100644
--- a/compiler/rustc_expand/src/mbe/transcribe.rs
+++ b/compiler/rustc_expand/src/mbe/transcribe.rs
@@ -367,7 +367,7 @@ impl LockstepIterSize {
 ///
 /// Example: `$($($x $y)+*);+` -- we need to make sure that `x` and `y` repeat the same amount as
 /// each other at the given depth when the macro was invoked. If they don't it might mean they were
-/// declared at unequal depths or there was a compile bug. For example, if we have 3 repetitions of
+/// declared at depths which weren't equal or there was a compiler bug. For example, if we have 3 repetitions of
 /// the outer sequence and 4 repetitions of the inner sequence for `x`, we should have the same for
 /// `y`; otherwise, we can't transcribe them both at the given depth.
 fn lockstep_iter_size(
diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs
index ddba14417..26bc216f6 100644
--- a/compiler/rustc_expand/src/proc_macro.rs
+++ b/compiler/rustc_expand/src/proc_macro.rs
@@ -54,7 +54,7 @@ impl base::BangProcMacro for BangProcMacro {
     ) -> Result<TokenStream, ErrorGuaranteed> {
         let _timer =
             ecx.sess.prof.generic_activity_with_arg_recorder("expand_proc_macro", |recorder| {
-                recorder.record_arg_with_span(ecx.expansion_descr(), span);
+                recorder.record_arg_with_span(ecx.sess.source_map(), ecx.expansion_descr(), span);
             });
 
         let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
@@ -85,7 +85,7 @@ impl base::AttrProcMacro for AttrProcMacro {
     ) -> Result<TokenStream, ErrorGuaranteed> {
         let _timer =
             ecx.sess.prof.generic_activity_with_arg_recorder("expand_proc_macro", |recorder| {
-                recorder.record_arg_with_span(ecx.expansion_descr(), span);
+                recorder.record_arg_with_span(ecx.sess.source_map(), ecx.expansion_descr(), span);
             });
 
         let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
@@ -134,7 +134,11 @@ impl MultiItemModifier for DeriveProcMacro {
         let stream = {
             let _timer =
                 ecx.sess.prof.generic_activity_with_arg_recorder("expand_proc_macro", |recorder| {
-                    recorder.record_arg_with_span(ecx.expansion_descr(), span);
+                    recorder.record_arg_with_span(
+                        ecx.sess.source_map(),
+                        ecx.expansion_descr(),
+                        span,
+                    );
                 });
             let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
             let strategy = exec_strategy(ecx);
diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs
index d304d38e1..3b9fc5e9a 100644
--- a/compiler/rustc_feature/src/accepted.rs
+++ b/compiler/rustc_feature/src/accepted.rs
@@ -90,7 +90,7 @@ declare_features! (
     (accepted, clone_closures, "1.26.0", Some(44490), None),
     /// Allows coercing non capturing closures to function pointers.
     (accepted, closure_to_fn_coercion, "1.19.0", Some(39817), None),
-    /// Allows using `cmpxchg16b` from `core::arch::x86_64`.
+    /// Allows using the CMPXCHG16B target feature.
     (accepted, cmpxchg16b_target_feature, "1.69.0", Some(44839), None),
     /// Allows usage of the `compile_error!` macro.
     (accepted, compile_error, "1.20.0", Some(40872), None),
@@ -238,6 +238,8 @@ declare_features! (
     (accepted, min_const_unsafe_fn, "1.33.0", Some(55607), None),
     /// Allows using `Self` and associated types in struct expressions and patterns.
     (accepted, more_struct_aliases, "1.16.0", Some(37544), None),
+    /// Allows using the MOVBE target feature.
+    (accepted, movbe_target_feature, "1.70.0", Some(44839), None),
     /// Allows patterns with concurrent by-move and by-ref bindings.
     /// For example, you can write `Foo(a, ref b)` where `a` is by-move and `b` is by-ref.
     (accepted, move_ref_pattern, "1.49.0", Some(68354), None),
diff --git a/compiler/rustc_feature/src/active.rs b/compiler/rustc_feature/src/active.rs
index baea5968e..052d312d9 100644
--- a/compiler/rustc_feature/src/active.rs
+++ b/compiler/rustc_feature/src/active.rs
@@ -160,6 +160,8 @@ declare_features! (
     (active, intrinsics, "1.0.0", None, None),
     /// Allows using `#[lang = ".."]` attribute for linking items to special compiler logic.
     (active, lang_items, "1.0.0", None, None),
+    /// Allows `#[link(..., cfg(..))]`; perma-unstable per #37406
+    (active, link_cfg, "1.14.0", None, None),
     /// Allows the `multiple_supertrait_upcastable` lint.
     (active, multiple_supertrait_upcastable, "1.69.0", None, None),
     /// Allows using `#[omit_gdb_pretty_printer_section]`.
@@ -200,8 +202,6 @@ declare_features! (
     (active, auto_traits, "1.50.0", Some(13231), None),
     /// Allows using `box` in patterns (RFC 469).
     (active, box_patterns, "1.0.0", Some(29641), None),
-    /// Allows using the `box $expr` syntax.
-    (active, box_syntax, "1.0.0", Some(49733), None),
     /// Allows `#[doc(notable_trait)]`.
     /// Renamed from `doc_spotlight`.
     (active, doc_notable_trait, "1.52.0", Some(45040), None),
@@ -225,7 +225,7 @@ declare_features! (
     (active, rustc_allow_const_fn_unstable, "1.49.0", Some(69399), None),
     /// Allows using compiler's own crates.
     (active, rustc_private, "1.0.0", Some(27812), None),
-    /// Allows using internal rustdoc features like `doc(primitive)` or `doc(keyword)`.
+    /// Allows using internal rustdoc features like `doc(keyword)`.
     (active, rustdoc_internals, "1.58.0", Some(90418), None),
     /// Allows using the `rustdoc::missing_doc_code_examples` lint
     (active, rustdoc_missing_doc_code_examples, "1.31.0", Some(101730), None),
@@ -259,7 +259,6 @@ declare_features! (
     (active, ermsb_target_feature, "1.49.0", Some(44839), None),
     (active, hexagon_target_feature, "1.27.0", Some(44839), None),
     (active, mips_target_feature, "1.27.0", Some(44839), None),
-    (active, movbe_target_feature, "1.34.0", Some(44839), None),
     (active, powerpc_target_feature, "1.27.0", Some(44839), None),
     (active, riscv_target_feature, "1.45.0", Some(44839), None),
     (active, rtm_target_feature, "1.35.0", Some(44839), None),
@@ -310,7 +309,7 @@ declare_features! (
     (active, associated_type_defaults, "1.2.0", Some(29661), None),
     /// Allows `async || body` closures.
     (active, async_closure, "1.37.0", Some(62290), None),
-    /// Alows async functions to be declared, implemented, and used in traits.
+    /// Allows async functions to be declared, implemented, and used in traits.
     (incomplete, async_fn_in_trait, "1.66.0", Some(91611), None),
     /// Allows `extern "C-unwind" fn` to enable unwinding across ABI boundaries.
     (active, c_unwind, "1.52.0", Some(74990), None),
@@ -417,6 +416,8 @@ declare_features! (
     (active, half_open_range_patterns_in_slices, "1.66.0", Some(67264), None),
     /// Allows `if let` guard in match arms.
     (active, if_let_guard, "1.47.0", Some(51114), None),
+    /// Allows `impl Trait` to be used inside associated types (RFC 2515).
+    (active, impl_trait_in_assoc_type, "1.70.0", Some(63063), None),
     /// Allows `impl Trait` as output type in `Fn` traits in return position of functions.
     (active, impl_trait_in_fn_trait_return, "1.64.0", Some(99697), None),
     /// Allows referencing `Self` and projections in impl-trait.
@@ -435,8 +436,6 @@ declare_features! (
     (active, large_assignments, "1.52.0", Some(83518), None),
     /// Allows `if/while p && let q = r && ...` chains.
     (active, let_chains, "1.37.0", Some(53667), None),
-    /// Allows `#[link(..., cfg(..))]`.
-    (active, link_cfg, "1.14.0", Some(37406), None),
     /// Allows using `reason` in lint attributes and the `#[expect(lint)]` lint check.
     (active, lint_reasons, "1.31.0", Some(54503), None),
     /// Give access to additional metadata about declarative macro meta-variables.
@@ -498,6 +497,8 @@ declare_features! (
     (active, repr_simd, "1.4.0", Some(27731), None),
     /// Allows return-position `impl Trait` in traits.
     (incomplete, return_position_impl_trait_in_trait, "1.65.0", Some(91611), None),
+    /// Allows bounding the return type of AFIT/RPITIT.
+    (incomplete, return_type_notation, "1.70.0", Some(109417), None),
     /// Allows `extern "rust-cold"`.
     (active, rust_cold_cc, "1.63.0", Some(97544), None),
     /// Allows the use of SIMD types in functions declared in `extern` blocks.
@@ -519,6 +520,8 @@ declare_features! (
     /// Allows dyn upcasting trait objects via supertraits.
     /// Dyn upcasting is casting, e.g., `dyn Foo -> dyn Bar` where `Foo: Bar`.
     (active, trait_upcasting, "1.56.0", Some(65991), None),
+    /// Allows for transmuting between arrays with sizes that contain generic consts.
+    (active, transmute_generic_consts, "1.70.0", Some(109929), None),
     /// Allows #[repr(transparent)] on unions (RFC 2645).
     (active, transparent_unions, "1.37.0", Some(60405), None),
     /// Allows inconsistent bounds in where clauses.
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 493a9cd89..c77292fdd 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -778,6 +778,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         definition of a trait, it's currently in experimental form and should be changed before \
         being exposed outside of the std"
     ),
+    rustc_attr!(
+        rustc_doc_primitive, Normal, template!(NameValueStr: "primitive name"), ErrorFollowing,
+        r#"`rustc_doc_primitive` is a rustc internal attribute"#,
+    ),
 
     // ==========================================================================
     // Internal attributes, Testing:
diff --git a/compiler/rustc_feature/src/lib.rs b/compiler/rustc_feature/src/lib.rs
index 93d167163..3ce16e156 100644
--- a/compiler/rustc_feature/src/lib.rs
+++ b/compiler/rustc_feature/src/lib.rs
@@ -11,7 +11,7 @@
 //! even if it is stabilized or removed, *do not remove it*. Instead, move the
 //! symbol to the `accepted` or `removed` modules respectively.
 
-#![feature(once_cell)]
+#![feature(lazy_cell)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index 04d4f6cb1..aa51eaafd 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -52,6 +52,8 @@ declare_features! (
     (removed, allow_fail, "1.19.0", Some(46488), None, Some("removed due to no clear use cases")),
     (removed, await_macro, "1.38.0", Some(50547), None,
      Some("subsumed by `.await` syntax")),
+    /// Allows using the `box $expr` syntax.
+    (removed, box_syntax, "1.70.0", Some(49733), None, Some("replaced with `#[rustc_box]`")),
     /// Allows capturing disjoint fields in a closure/generator (RFC 2229).
     (removed, capture_disjoint_fields, "1.49.0", Some(53488), None, Some("stabilized in Rust 2021")),
     /// Allows comparing raw pointers during const eval.
diff --git a/compiler/rustc_fs_util/src/lib.rs b/compiler/rustc_fs_util/src/lib.rs
index a7dfce3b9..81d633381 100644
--- a/compiler/rustc_fs_util/src/lib.rs
+++ b/compiler/rustc_fs_util/src/lib.rs
@@ -1,10 +1,11 @@
+#![feature(absolute_path)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
 use std::ffi::CString;
 use std::fs;
 use std::io;
-use std::path::{Path, PathBuf};
+use std::path::{absolute, Path, PathBuf};
 
 // Unfortunately, on windows, it looks like msvcrt.dll is silently translating
 // verbatim paths under the hood to non-verbatim paths! This manifests itself as
@@ -91,3 +92,8 @@ pub fn path_to_c_string(p: &Path) -> CString {
 pub fn path_to_c_string(p: &Path) -> CString {
     CString::new(p.to_str().unwrap()).unwrap()
 }
+
+#[inline]
+pub fn try_canonicalize<P: AsRef<Path>>(path: P) -> io::Result<PathBuf> {
+    fs::canonicalize(&path).or_else(|_| absolute(&path))
+}
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index 19d3d41c9..35a72f868 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -328,7 +328,7 @@ pub struct GenericArgs<'hir> {
     /// Were arguments written in parenthesized form `Fn(T) -> U`?
     /// This is required mostly for pretty-printing and diagnostics,
     /// but also for changing lifetime elision rules to be "function-like".
-    pub parenthesized: bool,
+    pub parenthesized: GenericArgsParentheses,
     /// The span encompassing arguments and the surrounding brackets `<>` or `()`
     ///       Foo<A, B, AssocTy = D>           Fn(T, U, V) -> W
     ///          ^^^^^^^^^^^^^^^^^^^             ^^^^^^^^^
@@ -340,11 +340,16 @@ pub struct GenericArgs<'hir> {
 
 impl<'hir> GenericArgs<'hir> {
     pub const fn none() -> Self {
-        Self { args: &[], bindings: &[], parenthesized: false, span_ext: DUMMY_SP }
+        Self {
+            args: &[],
+            bindings: &[],
+            parenthesized: GenericArgsParentheses::No,
+            span_ext: DUMMY_SP,
+        }
     }
 
     pub fn inputs(&self) -> &[Ty<'hir>] {
-        if self.parenthesized {
+        if self.parenthesized == GenericArgsParentheses::ParenSugar {
             for arg in self.args {
                 match arg {
                     GenericArg::Lifetime(_) => {}
@@ -417,6 +422,17 @@ impl<'hir> GenericArgs<'hir> {
     }
 }
 
+#[derive(Copy, Clone, PartialEq, Eq, Encodable, Hash, Debug)]
+#[derive(HashStable_Generic)]
+pub enum GenericArgsParentheses {
+    No,
+    /// Bounds for `feature(return_type_notation)`, like `T: Trait<method(..): Send>`,
+    /// where the args are explicitly elided with `..`
+    ReturnTypeNotation,
+    /// parenthesized function-family traits, like `T: Fn(u32) -> i32`
+    ParenSugar,
+}
+
 /// A modifier on a bound, currently this is only used for `?Sized`, where the
 /// modifier is `Maybe`. Negative bounds should also be handled here.
 #[derive(Copy, Clone, PartialEq, Eq, Encodable, Hash, Debug)]
@@ -815,12 +831,13 @@ pub struct ParentedNode<'tcx> {
 #[derive(Debug)]
 pub struct AttributeMap<'tcx> {
     pub map: SortedMap<ItemLocalId, &'tcx [Attribute]>,
-    pub hash: Fingerprint,
+    // Only present when the crate hash is needed.
+    pub opt_hash: Option<Fingerprint>,
 }
 
 impl<'tcx> AttributeMap<'tcx> {
     pub const EMPTY: &'static AttributeMap<'static> =
-        &AttributeMap { map: SortedMap::new(), hash: Fingerprint::ZERO };
+        &AttributeMap { map: SortedMap::new(), opt_hash: Some(Fingerprint::ZERO) };
 
     #[inline]
     pub fn get(&self, id: ItemLocalId) -> &'tcx [Attribute] {
@@ -832,10 +849,9 @@ impl<'tcx> AttributeMap<'tcx> {
 /// These nodes are mapped by `ItemLocalId` alongside the index of their parent node.
 /// The HIR tree, including bodies, is pre-hashed.
 pub struct OwnerNodes<'tcx> {
-    /// Pre-computed hash of the full HIR.
-    pub hash_including_bodies: Fingerprint,
-    /// Pre-computed hash of the item signature, without recursing into the body.
-    pub hash_without_bodies: Fingerprint,
+    /// Pre-computed hash of the full HIR. Used in the crate hash. Only present
+    /// when incr. comp. is enabled.
+    pub opt_hash_including_bodies: Option<Fingerprint>,
     /// Full HIR for the current owner.
     // The zeroth node's parent should never be accessed: the owner's parent is computed by the
     // hir_owner_parent query. It is set to `ItemLocalId::INVALID` to force an ICE if accidentally
@@ -872,8 +888,7 @@ impl fmt::Debug for OwnerNodes<'_> {
                     .collect::<Vec<_>>(),
             )
             .field("bodies", &self.bodies)
-            .field("hash_without_bodies", &self.hash_without_bodies)
-            .field("hash_including_bodies", &self.hash_including_bodies)
+            .field("opt_hash_including_bodies", &self.opt_hash_including_bodies)
             .finish()
     }
 }
@@ -940,7 +955,8 @@ impl<T> MaybeOwner<T> {
 #[derive(Debug)]
 pub struct Crate<'hir> {
     pub owners: IndexVec<LocalDefId, MaybeOwner<&'hir OwnerInfo<'hir>>>,
-    pub hir_hash: Fingerprint,
+    // Only present when incr. comp. is enabled.
+    pub opt_hir_hash: Option<Fingerprint>,
 }
 
 #[derive(Debug, HashStable_Generic)]
@@ -1673,7 +1689,6 @@ pub struct Expr<'hir> {
 impl Expr<'_> {
     pub fn precedence(&self) -> ExprPrecedence {
         match self.kind {
-            ExprKind::Box(_) => ExprPrecedence::Box,
             ExprKind::ConstBlock(_) => ExprPrecedence::ConstBlock,
             ExprKind::Array(_) => ExprPrecedence::Array,
             ExprKind::Call(..) => ExprPrecedence::Call,
@@ -1763,7 +1778,6 @@ impl Expr<'_> {
             | ExprKind::Lit(_)
             | ExprKind::ConstBlock(..)
             | ExprKind::Unary(..)
-            | ExprKind::Box(..)
             | ExprKind::AddrOf(..)
             | ExprKind::Binary(..)
             | ExprKind::Yield(..)
@@ -1851,7 +1865,6 @@ impl Expr<'_> {
             | ExprKind::InlineAsm(..)
             | ExprKind::AssignOp(..)
             | ExprKind::ConstBlock(..)
-            | ExprKind::Box(..)
             | ExprKind::Binary(..)
             | ExprKind::Yield(..)
             | ExprKind::DropTemps(..)
@@ -1862,8 +1875,7 @@ impl Expr<'_> {
     /// To a first-order approximation, is this a pattern?
     pub fn is_approximately_pattern(&self) -> bool {
         match &self.kind {
-            ExprKind::Box(_)
-            | ExprKind::Array(_)
+            ExprKind::Array(_)
             | ExprKind::Call(..)
             | ExprKind::Tup(_)
             | ExprKind::Lit(_)
@@ -1910,8 +1922,6 @@ pub fn is_range_literal(expr: &Expr<'_>) -> bool {
 
 #[derive(Debug, HashStable_Generic)]
 pub enum ExprKind<'hir> {
-    /// A `box x` expression.
-    Box(&'hir Expr<'hir>),
     /// Allow anonymous constants from an inline `const` block
     ConstBlock(AnonConst),
     /// An array (e.g., `[a, b, c, d]`).
diff --git a/compiler/rustc_hir/src/hir_id.rs b/compiler/rustc_hir/src/hir_id.rs
index 404abe2b0..37ac37231 100644
--- a/compiler/rustc_hir/src/hir_id.rs
+++ b/compiler/rustc_hir/src/hir_id.rs
@@ -22,6 +22,12 @@ impl From<OwnerId> for HirId {
     }
 }
 
+impl From<OwnerId> for DefId {
+    fn from(value: OwnerId) -> Self {
+        value.to_def_id()
+    }
+}
+
 impl OwnerId {
     #[inline]
     pub fn to_def_id(self) -> DefId {
diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs
index cc0f64017..234256ab5 100644
--- a/compiler/rustc_hir/src/intravisit.rs
+++ b/compiler/rustc_hir/src/intravisit.rs
@@ -682,7 +682,6 @@ pub fn walk_anon_const<'v, V: Visitor<'v>>(visitor: &mut V, constant: &'v AnonCo
 pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>) {
     visitor.visit_id(expression.hir_id);
     match expression.kind {
-        ExprKind::Box(ref subexpression) => visitor.visit_expr(subexpression),
         ExprKind::Array(subexpressions) => {
             walk_list!(visitor, visit_expr, subexpressions);
         }
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 60fa5a99e..8f91a96f9 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -49,7 +49,7 @@ impl LanguageItems {
         self.get(it).ok_or_else(|| LangItemError(it))
     }
 
-    pub fn iter<'a>(&'a self) -> impl Iterator<Item = (LangItem, DefId)> + 'a {
+    pub fn iter(&self) -> impl Iterator<Item = (LangItem, DefId)> + '_ {
         self.items
             .iter()
             .enumerate()
@@ -166,6 +166,9 @@ language_item_table! {
 
     Freeze,                  sym::freeze,              freeze_trait,               Target::Trait,          GenericRequirement::Exact(0);
 
+    FnPtrTrait,              sym::fn_ptr_trait,        fn_ptr_trait,               Target::Trait,          GenericRequirement::Exact(0);
+    FnPtrAddr,               sym::fn_ptr_addr,         fn_ptr_addr,                Target::Method(MethodKind::Trait { body: false }), GenericRequirement::None;
+
     Drop,                    sym::drop,                drop_trait,                 Target::Trait,          GenericRequirement::None;
     Destruct,                sym::destruct,            destruct_trait,             Target::Trait,          GenericRequirement::None;
 
@@ -237,6 +240,7 @@ language_item_table! {
     PanicDisplay,            sym::panic_display,       panic_display,              Target::Fn,             GenericRequirement::None;
     ConstPanicFmt,           sym::const_panic_fmt,     const_panic_fmt,            Target::Fn,             GenericRequirement::None;
     PanicBoundsCheck,        sym::panic_bounds_check,  panic_bounds_check_fn,      Target::Fn,             GenericRequirement::Exact(0);
+    PanicMisalignedPointerDereference,        sym::panic_misaligned_pointer_dereference,  panic_misaligned_pointer_dereference_fn,      Target::Fn,             GenericRequirement::Exact(0);
     PanicInfo,               sym::panic_info,          panic_info,                 Target::Struct,         GenericRequirement::None;
     PanicLocation,           sym::panic_location,      panic_location,             Target::Struct,         GenericRequirement::None;
     PanicImpl,               sym::panic_impl,          panic_impl,                 Target::Fn,             GenericRequirement::None;
@@ -296,12 +300,12 @@ language_item_table! {
     // FIXME(swatinem): the following lang items are used for async lowering and
     // should become obsolete eventually.
     ResumeTy,                sym::ResumeTy,            resume_ty,                  Target::Struct,         GenericRequirement::None;
-    IdentityFuture,          sym::identity_future,     identity_future_fn,         Target::Fn,             GenericRequirement::None;
     GetContext,              sym::get_context,         get_context_fn,             Target::Fn,             GenericRequirement::None;
 
     Context,                 sym::Context,             context,                    Target::Struct,         GenericRequirement::None;
     FuturePoll,              sym::poll,                future_poll_fn,             Target::Method(MethodKind::Trait { body: false }), GenericRequirement::None;
 
+    Option,                  sym::Option,              option_type,                Target::Enum,           GenericRequirement::None;
     OptionSome,              sym::Some,                option_some_variant,        Target::Variant,        GenericRequirement::None;
     OptionNone,              sym::None,                option_none_variant,        Target::Variant,        GenericRequirement::None;
 
diff --git a/compiler/rustc_hir/src/pat_util.rs b/compiler/rustc_hir/src/pat_util.rs
index e870aa543..838c123f8 100644
--- a/compiler/rustc_hir/src/pat_util.rs
+++ b/compiler/rustc_hir/src/pat_util.rs
@@ -2,7 +2,6 @@ use crate::def::{CtorOf, DefKind, Res};
 use crate::def_id::DefId;
 use crate::hir::{self, BindingAnnotation, ByRef, HirId, PatKind};
 use rustc_data_structures::fx::FxHashSet;
-use rustc_span::hygiene::DesugaringKind;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
@@ -136,14 +135,4 @@ impl hir::Pat<'_> {
         });
         result
     }
-
-    /// If the pattern is `Some(<pat>)` from a desugared for loop, returns the inner pattern
-    pub fn for_loop_some(&self) -> Option<&Self> {
-        if self.span.desugaring_kind() == Some(DesugaringKind::ForLoop) {
-            if let hir::PatKind::Struct(_, [pat_field], _) = self.kind {
-                return Some(pat_field.pat);
-            }
-        }
-        None
-    }
 }
diff --git a/compiler/rustc_hir/src/stable_hash_impls.rs b/compiler/rustc_hir/src/stable_hash_impls.rs
index 85d0e02d0..97fa710b3 100644
--- a/compiler/rustc_hir/src/stable_hash_impls.rs
+++ b/compiler/rustc_hir/src/stable_hash_impls.rs
@@ -100,24 +100,23 @@ impl<'tcx, HirCtx: crate::HashStableContext> HashStable<HirCtx> for OwnerNodes<'
         // `local_id_to_def_id` is also ignored because is dependent on the body, then just hashing
         // the body satisfies the condition of two nodes being different have different
         // `hash_stable` results.
-        let OwnerNodes { hash_including_bodies, hash_without_bodies: _, nodes: _, bodies: _ } =
-            *self;
-        hash_including_bodies.hash_stable(hcx, hasher);
+        let OwnerNodes { opt_hash_including_bodies, nodes: _, bodies: _ } = *self;
+        opt_hash_including_bodies.unwrap().hash_stable(hcx, hasher);
     }
 }
 
 impl<'tcx, HirCtx: crate::HashStableContext> HashStable<HirCtx> for AttributeMap<'tcx> {
     fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) {
-        // We ignore the `map` since it refers to information included in `hash` which is hashed in
-        // the collector and used for the crate hash.
-        let AttributeMap { hash, map: _ } = *self;
-        hash.hash_stable(hcx, hasher);
+        // We ignore the `map` since it refers to information included in `opt_hash` which is
+        // hashed in the collector and used for the crate hash.
+        let AttributeMap { opt_hash, map: _ } = *self;
+        opt_hash.unwrap().hash_stable(hcx, hasher);
     }
 }
 
 impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for Crate<'_> {
     fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) {
-        let Crate { owners: _, hir_hash } = self;
-        hir_hash.hash_stable(hcx, hasher)
+        let Crate { owners: _, opt_hir_hash } = self;
+        opt_hir_hash.unwrap().hash_stable(hcx, hasher)
     }
 }
diff --git a/compiler/rustc_hir_analysis/locales/en-US.ftl b/compiler/rustc_hir_analysis/messages.ftl
index 1d313945b..1d7965ff5 100644
--- a/compiler/rustc_hir_analysis/locales/en-US.ftl
+++ b/compiler/rustc_hir_analysis/messages.ftl
@@ -42,6 +42,9 @@ hir_analysis_assoc_type_binding_not_allowed =
     associated type bindings are not allowed here
     .label = associated type not allowed here
 
+hir_analysis_parenthesized_fn_trait_expansion =
+    parenthesized trait syntax expands to `{$expanded_type}`
+
 hir_analysis_typeof_reserved_keyword_used =
     `typeof` is a reserved keyword but unimplemented
     .suggestion = consider replacing `typeof(...)` with an actual type
@@ -89,14 +92,14 @@ hir_analysis_missing_type_params =
     .note = because of the default `Self` reference, type parameters must be specified on object types
 
 hir_analysis_copy_impl_on_type_with_dtor =
-    the trait `Copy` may not be implemented for this type; the type has a destructor
+    the trait `Copy` cannot be implemented for this type; the type has a destructor
     .label = `Copy` not allowed on types with destructors
 
 hir_analysis_multiple_relaxed_default_bounds =
     type parameter has more than one relaxed default bound, only one is supported
 
 hir_analysis_copy_impl_on_non_adt =
-    the trait `Copy` may not be implemented for this type
+    the trait `Copy` cannot be implemented for this type
     .label = type is not a structure or enumeration
 
 hir_analysis_const_impl_for_non_const_trait =
@@ -125,9 +128,14 @@ hir_analysis_where_clause_on_main = `main` function is not allowed to have a `wh
 hir_analysis_track_caller_on_main = `main` function is not allowed to be `#[track_caller]`
     .suggestion = remove this annotation
 
+hir_analysis_target_feature_on_main = `main` function is not allowed to have `#[target_feature]`
+
 hir_analysis_start_not_track_caller = `start` is not allowed to be `#[track_caller]`
     .label = `start` is not allowed to be `#[track_caller]`
 
+hir_analysis_start_not_target_feature = `start` is not allowed to have `#[target_feature]`
+    .label = `start` is not allowed to have `#[target_feature]`
+
 hir_analysis_start_not_async = `start` is not allowed to be `async`
     .label = `start` is not allowed to be `async`
 
@@ -155,3 +163,65 @@ hir_analysis_cannot_capture_late_bound_ty_in_anon_const =
 hir_analysis_cannot_capture_late_bound_const_in_anon_const =
     cannot capture late-bound const parameter in a constant
     .label = parameter defined here
+
+hir_analysis_variances_of = {$variances_of}
+
+hir_analysis_pass_to_variadic_function = can't pass `{$ty}` to variadic function
+    .suggestion = cast the value to `{$cast_ty}`
+    .help = cast the value to `{$cast_ty}`
+
+hir_analysis_cast_thin_pointer_to_fat_pointer = cannot cast thin pointer `{$expr_ty}` to fat pointer `{$cast_ty}`
+
+hir_analysis_invalid_union_field =
+    field must implement `Copy` or be wrapped in `ManuallyDrop<...>` to be used in a union
+    .note = union fields must not have drop side-effects, which is currently enforced via either `Copy` or `ManuallyDrop<...>`
+
+hir_analysis_invalid_union_field_sugg =
+    wrap the field type in `ManuallyDrop<...>`
+
+hir_analysis_return_type_notation_on_non_rpitit =
+    return type notation used on function that is not `async` and does not return `impl Trait`
+    .note = function returns `{$ty}`, which is not compatible with associated type return bounds
+    .label = this function must be `async` or return `impl Trait`
+
+hir_analysis_return_type_notation_equality_bound =
+    return type notation is not allowed to use type equality
+
+hir_analysis_return_type_notation_missing_method =
+    cannot find associated function `{$assoc_name}` in trait `{$trait_name}`
+
+hir_analysis_placeholder_not_allowed_item_signatures = the placeholder `_` is not allowed within types on item signatures for {$kind}
+    .label = not allowed in type signatures
+
+hir_analysis_associated_type_trait_uninferred_generic_params = cannot use the associated type of a trait with uninferred generic parameters
+    .suggestion = use a fully qualified path with inferred lifetimes
+
+hir_analysis_associated_type_trait_uninferred_generic_params_multipart_suggestion = use a fully qualified path with explicit lifetimes
+
+hir_analysis_enum_discriminant_overflowed = enum discriminant overflowed
+    .label = overflowed on value after {$discr}
+    .note = explicitly set `{$item_name} = {$wrapped_discr}` if that is desired outcome
+
+hir_analysis_paren_sugar_attribute = the `#[rustc_paren_sugar]` attribute is a temporary means of controlling which traits can use parenthetical notation
+    .help = add `#![feature(unboxed_closures)]` to the crate attributes to use it
+
+hir_analysis_must_implement_one_of_attribute = the `#[rustc_must_implement_one_of]` attribute must be used with at least 2 args
+
+hir_analysis_must_be_name_of_associated_function = must be a name of an associated function
+
+hir_analysis_function_not_have_default_implementation = function doesn't have a default implementation
+    .note = required by this annotation
+
+hir_analysis_must_implement_not_function = not a function
+
+hir_analysis_must_implement_not_function_span_note = required by this annotation
+
+hir_analysis_must_implement_not_function_note = all `#[rustc_must_implement_one_of]` arguments must be associated function names
+
+hir_analysis_function_not_found_in_trait = function not found in this trait
+
+hir_analysis_functions_names_duplicated = functions names are duplicated
+    .note = all `#[rustc_must_implement_one_of]` arguments must be unique
+
+hir_analysis_simd_ffi_highly_experimental = use of SIMD type{$snip} in FFI is highly experimental and may result in invalid code
+    .help = add `#![feature(simd_ffi)]` to the crate attributes to enable
diff --git a/compiler/rustc_hir_analysis/src/astconv/errors.rs b/compiler/rustc_hir_analysis/src/astconv/errors.rs
index c49e4d9d5..113c3f08a 100644
--- a/compiler/rustc_hir_analysis/src/astconv/errors.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/errors.rs
@@ -1,10 +1,14 @@
 use crate::astconv::AstConv;
-use crate::errors::{ManualImplementation, MissingTypeParams};
+use crate::errors::{
+    AssocTypeBindingNotAllowed, ManualImplementation, MissingTypeParams,
+    ParenthesizedFnTraitExpansion,
+};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{pluralize, struct_span_err, Applicability, Diagnostic, ErrorGuaranteed};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_infer::traits::FulfillmentError;
+use rustc_middle::ty::TyCtxt;
 use rustc_middle::ty::{self, Ty};
 use rustc_session::parse::feature_err;
 use rustc_span::edit_distance::find_best_match_for_name;
@@ -51,7 +55,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         let trait_def = self.tcx().trait_def(trait_def_id);
         if !trait_def.paren_sugar {
-            if trait_segment.args().parenthesized {
+            if trait_segment.args().parenthesized == hir::GenericArgsParentheses::ParenSugar {
                 // For now, require that parenthetical notation be used only with `Fn()` etc.
                 let mut err = feature_err(
                     &self.tcx().sess.parse_sess,
@@ -67,7 +71,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         let sess = self.tcx().sess;
 
-        if !trait_segment.args().parenthesized {
+        if trait_segment.args().parenthesized != hir::GenericArgsParentheses::ParenSugar {
             // For now, require that parenthetical notation be used only with `Fn()` etc.
             let mut err = feature_err(
                 &sess.parse_sess,
@@ -78,43 +82,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             // Do not suggest the other syntax if we are in trait impl:
             // the desugaring would contain an associated type constraint.
             if !is_impl {
-                let args = trait_segment
-                    .args
-                    .as_ref()
-                    .and_then(|args| args.args.get(0))
-                    .and_then(|arg| match arg {
-                        hir::GenericArg::Type(ty) => match ty.kind {
-                            hir::TyKind::Tup(t) => t
-                                .iter()
-                                .map(|e| sess.source_map().span_to_snippet(e.span))
-                                .collect::<Result<Vec<_>, _>>()
-                                .map(|a| a.join(", ")),
-                            _ => sess.source_map().span_to_snippet(ty.span),
-                        }
-                        .map(|s| format!("({})", s))
-                        .ok(),
-                        _ => None,
-                    })
-                    .unwrap_or_else(|| "()".to_string());
-                let ret = trait_segment
-                    .args()
-                    .bindings
-                    .iter()
-                    .find_map(|b| match (b.ident.name == sym::Output, &b.kind) {
-                        (true, hir::TypeBindingKind::Equality { term }) => {
-                            let span = match term {
-                                hir::Term::Ty(ty) => ty.span,
-                                hir::Term::Const(c) => self.tcx().hir().span(c.hir_id),
-                            };
-                            sess.source_map().span_to_snippet(span).ok()
-                        }
-                        _ => None,
-                    })
-                    .unwrap_or_else(|| "()".to_string());
                 err.span_suggestion(
                     span,
                     "use parenthetical notation instead",
-                    format!("{}{} -> {}", trait_segment.ident, args, ret),
+                    fn_trait_to_string(self.tcx(), trait_segment, true),
                     Applicability::MaybeIncorrect,
                 );
             }
@@ -512,8 +483,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 [segment] if segment.args.is_none() => {
                     trait_bound_spans = vec![segment.ident.span];
                     associated_types = associated_types
-                        .into_iter()
-                        .map(|(_, items)| (segment.ident.span, items))
+                        .into_values()
+                        .map(|items| (segment.ident.span, items))
                         .collect();
                 }
                 _ => {}
@@ -629,3 +600,77 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         err.emit();
     }
 }
+
+/// Emits an error regarding forbidden type binding associations
+pub fn prohibit_assoc_ty_binding(
+    tcx: TyCtxt<'_>,
+    span: Span,
+    segment: Option<(&hir::PathSegment<'_>, Span)>,
+) {
+    tcx.sess.emit_err(AssocTypeBindingNotAllowed {
+        span,
+        fn_trait_expansion: if let Some((segment, span)) = segment
+            && segment.args().parenthesized == hir::GenericArgsParentheses::ParenSugar
+        {
+            Some(ParenthesizedFnTraitExpansion {
+                span,
+                expanded_type: fn_trait_to_string(tcx, segment, false),
+            })
+        } else {
+            None
+        },
+    });
+}
+
+pub(crate) fn fn_trait_to_string(
+    tcx: TyCtxt<'_>,
+    trait_segment: &hir::PathSegment<'_>,
+    parenthesized: bool,
+) -> String {
+    let args = trait_segment
+        .args
+        .as_ref()
+        .and_then(|args| args.args.get(0))
+        .and_then(|arg| match arg {
+            hir::GenericArg::Type(ty) => match ty.kind {
+                hir::TyKind::Tup(t) => t
+                    .iter()
+                    .map(|e| tcx.sess.source_map().span_to_snippet(e.span))
+                    .collect::<Result<Vec<_>, _>>()
+                    .map(|a| a.join(", ")),
+                _ => tcx.sess.source_map().span_to_snippet(ty.span),
+            }
+            .map(|s| {
+                // `s.empty()` checks to see if the type is the unit tuple, if so we don't want a comma
+                if parenthesized || s.is_empty() { format!("({})", s) } else { format!("({},)", s) }
+            })
+            .ok(),
+            _ => None,
+        })
+        .unwrap_or_else(|| "()".to_string());
+
+    let ret = trait_segment
+        .args()
+        .bindings
+        .iter()
+        .find_map(|b| match (b.ident.name == sym::Output, &b.kind) {
+            (true, hir::TypeBindingKind::Equality { term }) => {
+                let span = match term {
+                    hir::Term::Ty(ty) => ty.span,
+                    hir::Term::Const(c) => tcx.hir().span(c.hir_id),
+                };
+
+                (span != tcx.hir().span(trait_segment.hir_id))
+                    .then_some(tcx.sess.source_map().span_to_snippet(span).ok())
+                    .flatten()
+            }
+            _ => None,
+        })
+        .unwrap_or_else(|| "()".to_string());
+
+    if parenthesized {
+        format!("{}{} -> {}", trait_segment.ident, args, ret)
+    } else {
+        format!("{}<{}, Output={}>", trait_segment.ident, args, ret)
+    }
+}
diff --git a/compiler/rustc_hir_analysis/src/astconv/generics.rs b/compiler/rustc_hir_analysis/src/astconv/generics.rs
index 7f6518ffd..3b5c67de2 100644
--- a/compiler/rustc_hir_analysis/src/astconv/generics.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/generics.rs
@@ -1,9 +1,8 @@
 use super::IsMethodCall;
 use crate::astconv::{
-    CreateSubstsForGenericArgsCtxt, ExplicitLateBound, GenericArgCountMismatch,
-    GenericArgCountResult, GenericArgPosition,
+    errors::prohibit_assoc_ty_binding, CreateSubstsForGenericArgsCtxt, ExplicitLateBound,
+    GenericArgCountMismatch, GenericArgCountResult, GenericArgPosition,
 };
-use crate::errors::AssocTypeBindingNotAllowed;
 use crate::structured_errors::{GenericArgsInfo, StructuredDiagnostic, WrongNumberOfGenericArgs};
 use rustc_ast::ast::ParamKindOrd;
 use rustc_errors::{struct_span_err, Applicability, Diagnostic, ErrorGuaranteed, MultiSpan};
@@ -433,7 +432,7 @@ pub(crate) fn check_generic_arg_count(
         (gen_pos != GenericArgPosition::Type || infer_args) && !gen_args.has_lifetime_params();
 
     if gen_pos != GenericArgPosition::Type && let Some(b) = gen_args.bindings.first() {
-            prohibit_assoc_ty_binding(tcx, b.span);
+             prohibit_assoc_ty_binding(tcx, b.span, None);
         }
 
     let explicit_late_bound =
@@ -589,11 +588,6 @@ pub(crate) fn check_generic_arg_count(
     }
 }
 
-/// Emits an error regarding forbidden type binding associations
-pub fn prohibit_assoc_ty_binding(tcx: TyCtxt<'_>, span: Span) {
-    tcx.sess.emit_err(AssocTypeBindingNotAllowed { span });
-}
-
 /// Prohibits explicit lifetime arguments if late-bound lifetime parameters
 /// are present. This is used both for datatypes and function calls.
 pub(crate) fn prohibit_explicit_late_bound_lifetimes(
@@ -618,7 +612,7 @@ pub(crate) fn prohibit_explicit_late_bound_lifetimes(
         if position == GenericArgPosition::Value
             && args.num_lifetime_params() != param_counts.lifetimes
         {
-            let mut err = tcx.sess.struct_span_err(span, msg);
+            let mut err = struct_span_err!(tcx.sess, span, E0794, "{}", msg);
             err.span_note(span_late, note);
             err.emit();
         } else {
diff --git a/compiler/rustc_hir_analysis/src/astconv/mod.rs b/compiler/rustc_hir_analysis/src/astconv/mod.rs
index f5ce02c9e..8d1156c17 100644
--- a/compiler/rustc_hir_analysis/src/astconv/mod.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/mod.rs
@@ -5,9 +5,8 @@
 mod errors;
 pub mod generics;
 
-use crate::astconv::generics::{
-    check_generic_arg_count, create_substs_for_generic_args, prohibit_assoc_ty_binding,
-};
+use crate::astconv::errors::prohibit_assoc_ty_binding;
+use crate::astconv::generics::{check_generic_arg_count, create_substs_for_generic_args};
 use crate::bounds::Bounds;
 use crate::collect::HirPlaceholderCollector;
 use crate::errors::{
@@ -32,10 +31,11 @@ use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
 use rustc_infer::traits::ObligationCause;
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
 use rustc_middle::middle::stability::AllowUnstable;
+use rustc_middle::ty::fold::FnMutDelegate;
 use rustc_middle::ty::subst::{self, GenericArgKind, InternalSubsts, SubstsRef};
-use rustc_middle::ty::DynKind;
 use rustc_middle::ty::GenericParamDefKind;
-use rustc_middle::ty::{self, Const, DefIdTree, IsSuggestable, Ty, TyCtxt, TypeVisitableExt};
+use rustc_middle::ty::{self, Const, IsSuggestable, Ty, TyCtxt, TypeVisitableExt};
+use rustc_middle::ty::{DynKind, ToPredicate};
 use rustc_session::lint::builtin::{AMBIGUOUS_ASSOCIATED_ITEMS, BARE_TRAIT_OBJECTS};
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
@@ -76,7 +76,7 @@ pub trait AstConv<'tcx> {
     fn get_type_parameter_bounds(
         &self,
         span: Span,
-        def_id: DefId,
+        def_id: LocalDefId,
         assoc_name: Ident,
     ) -> ty::GenericPredicates<'tcx>;
 
@@ -295,7 +295,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             ty::BoundConstness::NotConst,
         );
         if let Some(b) = item_segment.args().bindings.first() {
-            prohibit_assoc_ty_binding(self.tcx(), b.span);
+            prohibit_assoc_ty_binding(self.tcx(), b.span, Some((item_segment, span)));
         }
 
         substs
@@ -631,7 +631,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         );
 
         if let Some(b) = item_segment.args().bindings.first() {
-            prohibit_assoc_ty_binding(self.tcx(), b.span);
+            prohibit_assoc_ty_binding(self.tcx(), b.span, Some((item_segment, span)));
         }
 
         args
@@ -825,7 +825,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             constness,
         );
         if let Some(b) = trait_segment.args().bindings.first() {
-            prohibit_assoc_ty_binding(self.tcx(), b.span);
+            prohibit_assoc_ty_binding(self.tcx(), b.span, Some((trait_segment, span)));
         }
         self.tcx().mk_trait_ref(trait_def_id, substs)
     }
@@ -854,16 +854,15 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         )
     }
 
-    fn trait_defines_associated_type_named(&self, trait_def_id: DefId, assoc_name: Ident) -> bool {
-        self.tcx()
-            .associated_items(trait_def_id)
-            .find_by_name_and_kind(self.tcx(), assoc_name, ty::AssocKind::Type, trait_def_id)
-            .is_some()
-    }
-    fn trait_defines_associated_const_named(&self, trait_def_id: DefId, assoc_name: Ident) -> bool {
+    fn trait_defines_associated_item_named(
+        &self,
+        trait_def_id: DefId,
+        assoc_kind: ty::AssocKind,
+        assoc_name: Ident,
+    ) -> bool {
         self.tcx()
             .associated_items(trait_def_id)
-            .find_by_name_and_kind(self.tcx(), assoc_name, ty::AssocKind::Const, trait_def_id)
+            .find_by_name_and_kind(self.tcx(), assoc_name, assoc_kind, trait_def_id)
             .is_some()
     }
 
@@ -1087,24 +1086,44 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         let tcx = self.tcx();
 
-        let candidate =
-            if self.trait_defines_associated_type_named(trait_ref.def_id(), binding.item_name) {
-                // Simple case: X is defined in the current trait.
+        let return_type_notation =
+            binding.gen_args.parenthesized == hir::GenericArgsParentheses::ReturnTypeNotation;
+
+        let candidate = if return_type_notation {
+            if self.trait_defines_associated_item_named(
+                trait_ref.def_id(),
+                ty::AssocKind::Fn,
+                binding.item_name,
+            ) {
                 trait_ref
             } else {
-                // Otherwise, we have to walk through the supertraits to find
-                // those that do.
-                self.one_bound_for_assoc_type(
-                    || traits::supertraits(tcx, trait_ref),
-                    trait_ref.print_only_trait_path(),
-                    binding.item_name,
-                    path_span,
-                    match binding.kind {
-                        ConvertedBindingKind::Equality(term) => Some(term),
-                        _ => None,
-                    },
-                )?
-            };
+                return Err(tcx.sess.emit_err(crate::errors::ReturnTypeNotationMissingMethod {
+                    span: binding.span,
+                    trait_name: tcx.item_name(trait_ref.def_id()),
+                    assoc_name: binding.item_name.name,
+                }));
+            }
+        } else if self.trait_defines_associated_item_named(
+            trait_ref.def_id(),
+            ty::AssocKind::Type,
+            binding.item_name,
+        ) {
+            // Simple case: X is defined in the current trait.
+            trait_ref
+        } else {
+            // Otherwise, we have to walk through the supertraits to find
+            // those that do.
+            self.one_bound_for_assoc_type(
+                || traits::supertraits(tcx, trait_ref),
+                trait_ref.print_only_trait_path(),
+                binding.item_name,
+                path_span,
+                match binding.kind {
+                    ConvertedBindingKind::Equality(term) => Some(term),
+                    _ => None,
+                },
+            )?
+        };
 
         let (assoc_ident, def_scope) =
             tcx.adjust_ident_and_get_scope(binding.item_name, candidate.def_id(), hir_ref_id);
@@ -1116,9 +1135,13 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 .filter_by_name_unhygienic(assoc_ident.name)
                 .find(|i| i.kind == kind && i.ident(tcx).normalize_to_macros_2_0() == assoc_ident)
         };
-        let assoc_item = find_item_of_kind(ty::AssocKind::Type)
-            .or_else(|| find_item_of_kind(ty::AssocKind::Const))
-            .expect("missing associated type");
+        let assoc_item = if return_type_notation {
+            find_item_of_kind(ty::AssocKind::Fn)
+        } else {
+            find_item_of_kind(ty::AssocKind::Type)
+                .or_else(|| find_item_of_kind(ty::AssocKind::Const))
+        }
+        .expect("missing associated type");
 
         if !assoc_item.visibility(tcx).is_accessible_from(def_scope, tcx) {
             tcx.sess
@@ -1135,7 +1158,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             dup_bindings
                 .entry(assoc_item.def_id)
                 .and_modify(|prev_span| {
-                    self.tcx().sess.emit_err(ValueOfAssociatedStructAlreadySpecified {
+                    tcx.sess.emit_err(ValueOfAssociatedStructAlreadySpecified {
                         span: binding.span,
                         prev_span: *prev_span,
                         item_name: binding.item_name,
@@ -1145,28 +1168,100 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 .or_insert(binding.span);
         }
 
-        // Include substitutions for generic parameters of associated types
-        let projection_ty = candidate.map_bound(|trait_ref| {
-            let ident = Ident::new(assoc_item.name, binding.item_name.span);
-            let item_segment = hir::PathSegment {
-                ident,
-                hir_id: binding.hir_id,
-                res: Res::Err,
-                args: Some(binding.gen_args),
-                infer_args: false,
+        let projection_ty = if return_type_notation {
+            // If we have an method return type bound, then we need to substitute
+            // the method's early bound params with suitable late-bound params.
+            let mut num_bound_vars = candidate.bound_vars().len();
+            let substs =
+                candidate.skip_binder().substs.extend_to(tcx, assoc_item.def_id, |param, _| {
+                    let subst = match param.kind {
+                        GenericParamDefKind::Lifetime => tcx
+                            .mk_re_late_bound(
+                                ty::INNERMOST,
+                                ty::BoundRegion {
+                                    var: ty::BoundVar::from_usize(num_bound_vars),
+                                    kind: ty::BoundRegionKind::BrNamed(param.def_id, param.name),
+                                },
+                            )
+                            .into(),
+                        GenericParamDefKind::Type { .. } => tcx
+                            .mk_bound(
+                                ty::INNERMOST,
+                                ty::BoundTy {
+                                    var: ty::BoundVar::from_usize(num_bound_vars),
+                                    kind: ty::BoundTyKind::Param(param.def_id, param.name),
+                                },
+                            )
+                            .into(),
+                        GenericParamDefKind::Const { .. } => {
+                            let ty = tcx
+                                .type_of(param.def_id)
+                                .no_bound_vars()
+                                .expect("ct params cannot have early bound vars");
+                            tcx.mk_const(
+                                ty::ConstKind::Bound(
+                                    ty::INNERMOST,
+                                    ty::BoundVar::from_usize(num_bound_vars),
+                                ),
+                                ty,
+                            )
+                            .into()
+                        }
+                    };
+                    num_bound_vars += 1;
+                    subst
+                });
+
+            // Next, we need to check that the return-type notation is being used on
+            // an RPITIT (return-position impl trait in trait) or AFIT (async fn in trait).
+            let output = tcx.fn_sig(assoc_item.def_id).skip_binder().output();
+            let output = if let ty::Alias(ty::Projection, alias_ty) = *output.skip_binder().kind()
+                && tcx.def_kind(alias_ty.def_id) == DefKind::ImplTraitPlaceholder
+            {
+                alias_ty
+            } else {
+                return Err(self.tcx().sess.emit_err(
+                    crate::errors::ReturnTypeNotationOnNonRpitit {
+                        span: binding.span,
+                        ty: tcx.liberate_late_bound_regions(assoc_item.def_id, output),
+                        fn_span: tcx.hir().span_if_local(assoc_item.def_id),
+                        note: (),
+                    },
+                ));
             };
 
-            let substs_trait_ref_and_assoc_item = self.create_substs_for_associated_item(
-                path_span,
-                assoc_item.def_id,
-                &item_segment,
-                trait_ref.substs,
-            );
+            // Finally, move the fn return type's bound vars over to account for the early bound
+            // params (and trait ref's late bound params). This logic is very similar to
+            // `Predicate::subst_supertrait`, and it's no coincidence why.
+            let shifted_output = tcx.shift_bound_var_indices(num_bound_vars, output);
+            let subst_output = ty::EarlyBinder(shifted_output).subst(tcx, substs);
+
+            let bound_vars = tcx.late_bound_vars(binding.hir_id);
+            ty::Binder::bind_with_vars(subst_output, bound_vars)
+        } else {
+            // Include substitutions for generic parameters of associated types
+            candidate.map_bound(|trait_ref| {
+                let ident = Ident::new(assoc_item.name, binding.item_name.span);
+                let item_segment = hir::PathSegment {
+                    ident,
+                    hir_id: binding.hir_id,
+                    res: Res::Err,
+                    args: Some(binding.gen_args),
+                    infer_args: false,
+                };
+
+                let substs_trait_ref_and_assoc_item = self.create_substs_for_associated_item(
+                    path_span,
+                    assoc_item.def_id,
+                    &item_segment,
+                    trait_ref.substs,
+                );
 
-            debug!(?substs_trait_ref_and_assoc_item);
+                debug!(?substs_trait_ref_and_assoc_item);
 
-            self.tcx().mk_alias_ty(assoc_item.def_id, substs_trait_ref_and_assoc_item)
-        });
+                tcx.mk_alias_ty(assoc_item.def_id, substs_trait_ref_and_assoc_item)
+            })
+        };
 
         if !speculative {
             // Find any late-bound regions declared in `ty` that are not
@@ -1206,6 +1301,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         }
 
         match binding.kind {
+            ConvertedBindingKind::Equality(..) if return_type_notation => {
+                return Err(self.tcx().sess.emit_err(
+                    crate::errors::ReturnTypeNotationEqualityBound { span: binding.span },
+                ));
+            }
             ConvertedBindingKind::Equality(mut term) => {
                 // "Desugar" a constraint like `T: Iterator<Item = u32>` this to
                 // the "projection predicate" for:
@@ -1267,7 +1367,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 // Calling `skip_binder` is okay, because `add_bounds` expects the `param_ty`
                 // parameter to have a skipped binder.
                 let param_ty = tcx.mk_alias(ty::Projection, projection_ty.skip_binder());
-                self.add_bounds(param_ty, ast_bounds.iter(), bounds, candidate.bound_vars());
+                self.add_bounds(param_ty, ast_bounds.iter(), bounds, projection_ty.bound_vars());
             }
         }
         Ok(())
@@ -1336,7 +1436,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     ty::Clause::RegionOutlives(_) | ty::Clause::ConstArgHasType(..) => bug!(),
                 },
                 ty::PredicateKind::WellFormed(_)
-                | ty::PredicateKind::AliasEq(..)
+                | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::ObjectSafe(_)
                 | ty::PredicateKind::ClosureKind(_, _, _)
                 | ty::PredicateKind::Subtype(_)
@@ -1426,14 +1526,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         for (base_trait_ref, span, constness) in regular_traits_refs_spans {
             assert_eq!(constness, ty::BoundConstness::NotConst);
+            let base_pred: ty::Predicate<'tcx> = base_trait_ref.to_predicate(tcx);
+            for pred in traits::elaborate(tcx, [base_pred]) {
+                debug!("conv_object_ty_poly_trait_ref: observing object predicate `{:?}`", pred);
 
-            for obligation in traits::elaborate_trait_ref(tcx, base_trait_ref) {
-                debug!(
-                    "conv_object_ty_poly_trait_ref: observing object predicate `{:?}`",
-                    obligation.predicate
-                );
-
-                let bound_predicate = obligation.predicate.kind();
+                let bound_predicate = pred.kind();
                 match bound_predicate.skip_binder() {
                     ty::PredicateKind::Clause(ty::Clause::Trait(pred)) => {
                         let pred = bound_predicate.rebind(pred);
@@ -1441,6 +1538,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             tcx.associated_items(pred.def_id())
                                 .in_definition_order()
                                 .filter(|item| item.kind == ty::AssocKind::Type)
+                                .filter(|item| tcx.opt_rpitit_info(item.def_id).is_none())
                                 .map(|item| item.def_id),
                         );
                     }
@@ -1565,39 +1663,45 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             })
         });
 
-        let existential_projections = projection_bounds.iter().map(|(bound, _)| {
-            bound.map_bound(|mut b| {
-                assert_eq!(b.projection_ty.self_ty(), dummy_self);
-
-                // Like for trait refs, verify that `dummy_self` did not leak inside default type
-                // parameters.
-                let references_self = b.projection_ty.substs.iter().skip(1).any(|arg| {
-                    if arg.walk().any(|arg| arg == dummy_self.into()) {
-                        return true;
+        let existential_projections = projection_bounds
+            .iter()
+            // We filter out traits that don't have `Self` as their self type above,
+            // we need to do the same for projections.
+            .filter(|(bound, _)| bound.skip_binder().self_ty() == dummy_self)
+            .map(|(bound, _)| {
+                bound.map_bound(|mut b| {
+                    assert_eq!(b.projection_ty.self_ty(), dummy_self);
+
+                    // Like for trait refs, verify that `dummy_self` did not leak inside default type
+                    // parameters.
+                    let references_self = b.projection_ty.substs.iter().skip(1).any(|arg| {
+                        if arg.walk().any(|arg| arg == dummy_self.into()) {
+                            return true;
+                        }
+                        false
+                    });
+                    if references_self {
+                        let guar = tcx.sess.delay_span_bug(
+                            span,
+                            "trait object projection bounds reference `Self`",
+                        );
+                        let substs: Vec<_> = b
+                            .projection_ty
+                            .substs
+                            .iter()
+                            .map(|arg| {
+                                if arg.walk().any(|arg| arg == dummy_self.into()) {
+                                    return tcx.ty_error(guar).into();
+                                }
+                                arg
+                            })
+                            .collect();
+                        b.projection_ty.substs = tcx.mk_substs(&substs);
                     }
-                    false
-                });
-                if references_self {
-                    let guar = tcx
-                        .sess
-                        .delay_span_bug(span, "trait object projection bounds reference `Self`");
-                    let substs: Vec<_> = b
-                        .projection_ty
-                        .substs
-                        .iter()
-                        .map(|arg| {
-                            if arg.walk().any(|arg| arg == dummy_self.into()) {
-                                return tcx.ty_error(guar).into();
-                            }
-                            arg
-                        })
-                        .collect();
-                    b.projection_ty.substs = tcx.mk_substs(&substs);
-                }
 
-                ty::ExistentialProjection::erase_self_ty(tcx, b)
-            })
-        });
+                    ty::ExistentialProjection::erase_self_ty(tcx, b)
+                })
+            });
 
         let regular_trait_predicates = existential_trait_refs
             .map(|trait_ref| trait_ref.map_bound(ty::ExistentialPredicate::Trait));
@@ -1773,9 +1877,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             ty_param_def_id, assoc_name, span,
         );
 
-        let predicates = &self
-            .get_type_parameter_bounds(span, ty_param_def_id.to_def_id(), assoc_name)
-            .predicates;
+        let predicates =
+            &self.get_type_parameter_bounds(span, ty_param_def_id, assoc_name).predicates;
 
         debug!("find_bound_for_assoc_item: predicates={:#?}", predicates);
 
@@ -1811,10 +1914,12 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     where
         I: Iterator<Item = ty::PolyTraitRef<'tcx>>,
     {
-        let mut matching_candidates = all_candidates()
-            .filter(|r| self.trait_defines_associated_type_named(r.def_id(), assoc_name));
-        let mut const_candidates = all_candidates()
-            .filter(|r| self.trait_defines_associated_const_named(r.def_id(), assoc_name));
+        let mut matching_candidates = all_candidates().filter(|r| {
+            self.trait_defines_associated_item_named(r.def_id(), ty::AssocKind::Type, assoc_name)
+        });
+        let mut const_candidates = all_candidates().filter(|r| {
+            self.trait_defines_associated_item_named(r.def_id(), ty::AssocKind::Const, assoc_name)
+        });
 
         let (bound, next_cand) = match (matching_candidates.next(), const_candidates.next()) {
             (Some(bound), _) => (bound, matching_candidates.next()),
@@ -2226,47 +2331,66 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         let param_env = tcx.param_env(block.owner.to_def_id());
         let cause = ObligationCause::misc(span, block.owner.def_id);
+
         let mut fulfillment_errors = Vec::new();
-        let mut applicable_candidates: Vec<_> = candidates
-            .iter()
-            .filter_map(|&(impl_, (assoc_item, def_scope))| {
-                infcx.probe(|_| {
-                    let ocx = ObligationCtxt::new_in_snapshot(&infcx);
+        let mut applicable_candidates: Vec<_> = infcx.probe(|_| {
+            let universe = infcx.create_next_universe();
+
+            // Regions are not considered during selection.
+            let self_ty = tcx.replace_escaping_bound_vars_uncached(
+                self_ty,
+                FnMutDelegate {
+                    regions: &mut |_| tcx.lifetimes.re_erased,
+                    types: &mut |bv| {
+                        tcx.mk_placeholder(ty::PlaceholderType { universe, bound: bv })
+                    },
+                    consts: &mut |bv, ty| {
+                        tcx.mk_const(ty::PlaceholderConst { universe, bound: bv }, ty)
+                    },
+                },
+            );
 
-                    let impl_ty = tcx.type_of(impl_);
-                    let impl_substs = infcx.fresh_item_substs(impl_);
-                    let impl_ty = impl_ty.subst(tcx, impl_substs);
-                    let impl_ty = ocx.normalize(&cause, param_env, impl_ty);
+            candidates
+                .iter()
+                .filter_map(|&(impl_, (assoc_item, def_scope))| {
+                    infcx.probe(|_| {
+                        let ocx = ObligationCtxt::new_in_snapshot(&infcx);
 
-                    // Check that the Self-types can be related.
-                    // FIXME(fmease): Should we use `eq` here?
-                    ocx.sup(&ObligationCause::dummy(), param_env, impl_ty, self_ty).ok()?;
+                        let impl_ty = tcx.type_of(impl_);
+                        let impl_substs = infcx.fresh_item_substs(impl_);
+                        let impl_ty = impl_ty.subst(tcx, impl_substs);
+                        let impl_ty = ocx.normalize(&cause, param_env, impl_ty);
 
-                    // Check whether the impl imposes obligations we have to worry about.
-                    let impl_bounds = tcx.predicates_of(impl_);
-                    let impl_bounds = impl_bounds.instantiate(tcx, impl_substs);
+                        // Check that the Self-types can be related.
+                        // FIXME(fmease): Should we use `eq` here?
+                        ocx.sup(&ObligationCause::dummy(), param_env, impl_ty, self_ty).ok()?;
 
-                    let impl_bounds = ocx.normalize(&cause, param_env, impl_bounds);
+                        // Check whether the impl imposes obligations we have to worry about.
+                        let impl_bounds = tcx.predicates_of(impl_);
+                        let impl_bounds = impl_bounds.instantiate(tcx, impl_substs);
 
-                    let impl_obligations = traits::predicates_for_generics(
-                        |_, _| cause.clone(),
-                        param_env,
-                        impl_bounds,
-                    );
+                        let impl_bounds = ocx.normalize(&cause, param_env, impl_bounds);
 
-                    ocx.register_obligations(impl_obligations);
+                        let impl_obligations = traits::predicates_for_generics(
+                            |_, _| cause.clone(),
+                            param_env,
+                            impl_bounds,
+                        );
 
-                    let mut errors = ocx.select_where_possible();
-                    if !errors.is_empty() {
-                        fulfillment_errors.append(&mut errors);
-                        return None;
-                    }
+                        ocx.register_obligations(impl_obligations);
+
+                        let mut errors = ocx.select_where_possible();
+                        if !errors.is_empty() {
+                            fulfillment_errors.append(&mut errors);
+                            return None;
+                        }
 
-                    // FIXME(fmease): Unsolved vars can escape this InferCtxt snapshot.
-                    Some((assoc_item, def_scope, infcx.resolve_vars_if_possible(impl_substs)))
+                        // FIXME(fmease): Unsolved vars can escape this InferCtxt snapshot.
+                        Some((assoc_item, def_scope, infcx.resolve_vars_if_possible(impl_substs)))
+                    })
                 })
-            })
-            .collect();
+                .collect()
+        });
 
         if applicable_candidates.len() > 1 {
             return Err(self.complain_about_ambiguous_inherent_assoc_type(
@@ -2396,13 +2520,16 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                                     tcx,
                                     infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id),
                                 );
+                                let value = tcx.fold_regions(qself_ty, |_, _| tcx.lifetimes.re_erased);
+                                // FIXME: Don't bother dealing with non-lifetime binders here...
+                                if value.has_escaping_bound_vars() {
+                                    return false;
+                                }
                                 infcx
                                     .can_eq(
                                         ty::ParamEnv::empty(),
                                         impl_.self_ty(),
-                                        // Must fold past escaping bound vars too,
-                                        // since we have those at this point in astconv.
-                                        tcx.fold_regions(qself_ty, |_, _| tcx.lifetimes.re_erased),
+                                        value,
                                     )
                             })
                             && tcx.impl_polarity(impl_def_id) != ty::ImplPolarity::Negative
@@ -2451,7 +2578,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 tcx.all_impls(trait_def_id)
                     .filter(|impl_def_id| {
                         // Consider only accessible traits
-                        tcx.visibility(*impl_def_id).is_accessible_from(self.item_def_id(), tcx)
+                        tcx.visibility(trait_def_id).is_accessible_from(self.item_def_id(), tcx)
                             && tcx.impl_polarity(impl_def_id) != ty::ImplPolarity::Negative
                     })
                     .filter_map(|impl_def_id| tcx.impl_trait_ref(impl_def_id))
@@ -2596,7 +2723,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         for segment in segments {
             // Only emit the first error to avoid overloading the user with error messages.
             if let Some(b) = segment.args().bindings.first() {
-                prohibit_assoc_ty_binding(self.tcx(), b.span);
+                prohibit_assoc_ty_binding(self.tcx(), b.span, None);
                 return true;
             }
         }
@@ -3049,10 +3176,18 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             }
             &hir::TyKind::OpaqueDef(item_id, lifetimes, in_trait) => {
                 let opaque_ty = tcx.hir().item(item_id);
-                let def_id = item_id.owner_id.to_def_id();
 
                 match opaque_ty.kind {
                     hir::ItemKind::OpaqueTy(hir::OpaqueTy { origin, .. }) => {
+                        let local_def_id = item_id.owner_id.def_id;
+                        // If this is an RPITIT and we are using the new RPITIT lowering scheme, we
+                        // generate the def_id of an associated type for the trait and return as
+                        // type a projection.
+                        let def_id = if in_trait && tcx.lower_impl_trait_in_trait_to_assoc_ty() {
+                            tcx.associated_type_for_impl_trait_in_trait(local_def_id).to_def_id()
+                        } else {
+                            local_def_id.to_def_id()
+                        };
                         self.impl_trait_ty_to_ty(def_id, lifetimes, origin, in_trait)
                     }
                     ref i => bug!("`impl Trait` pointed to non-opaque type?? {:#?}", i),
@@ -3133,8 +3268,12 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         debug!("impl_trait_ty_to_ty: generics={:?}", generics);
         let substs = InternalSubsts::for_item(tcx, def_id, |param, _| {
-            if let Some(i) = (param.index as usize).checked_sub(generics.parent_count) {
-                // Our own parameters are the resolved lifetimes.
+            // We use `generics.count() - lifetimes.len()` here instead of `generics.parent_count`
+            // since return-position impl trait in trait squashes all of the generics from its source fn
+            // into its own generics, so the opaque's "own" params isn't always just lifetimes.
+            if let Some(i) = (param.index as usize).checked_sub(generics.count() - lifetimes.len())
+            {
+                // Resolve our own lifetime parameters.
                 let GenericParamDefKind::Lifetime { .. } = param.kind else { bug!() };
                 let hir::GenericArg::Lifetime(lifetime) = &lifetimes[i] else { bug!() };
                 self.ast_region_to_region(lifetime, None).into()
@@ -3309,10 +3448,13 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             tcx,
             trait_ref.substs.extend_to(tcx, assoc.def_id, |param, _| tcx.mk_param_from_def(param)),
         );
+        let fn_sig = tcx.liberate_late_bound_regions(fn_hir_id.expect_owner().to_def_id(), fn_sig);
 
-        let ty = if let Some(arg_idx) = arg_idx { fn_sig.input(arg_idx) } else { fn_sig.output() };
-
-        Some(tcx.liberate_late_bound_regions(fn_hir_id.expect_owner().to_def_id(), ty))
+        Some(if let Some(arg_idx) = arg_idx {
+            *fn_sig.inputs().get(arg_idx)?
+        } else {
+            fn_sig.output()
+        })
     }
 
     #[instrument(level = "trace", skip(self, generate_err))]
diff --git a/compiler/rustc_hir_analysis/src/bounds.rs b/compiler/rustc_hir_analysis/src/bounds.rs
index 0880c8c15..284b099e7 100644
--- a/compiler/rustc_hir_analysis/src/bounds.rs
+++ b/compiler/rustc_hir_analysis/src/bounds.rs
@@ -58,7 +58,7 @@ impl<'tcx> Bounds<'tcx> {
     pub fn push_sized(&mut self, tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, span: Span) {
         let sized_def_id = tcx.require_lang_item(LangItem::Sized, Some(span));
         let trait_ref = ty::Binder::dummy(tcx.mk_trait_ref(sized_def_id, [ty]));
-        // Preferrable to put this obligation first, since we report better errors for sized ambiguity.
+        // Preferable to put this obligation first, since we report better errors for sized ambiguity.
         self.predicates.insert(0, (trait_ref.without_const().to_predicate(tcx), span));
     }
 
diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index 848828175..0bb98fdf2 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -1,5 +1,5 @@
 use crate::check::intrinsicck::InlineAsmCtxt;
-use crate::errors::LinkageType;
+use crate::errors::{self, LinkageType};
 
 use super::compare_impl_item::check_type_bounds;
 use super::compare_impl_item::{compare_impl_method, compare_impl_ty};
@@ -22,12 +22,12 @@ use rustc_middle::ty::layout::{LayoutError, MAX_SIMD_LANES};
 use rustc_middle::ty::subst::GenericArgKind;
 use rustc_middle::ty::util::{Discr, IntTypeExt};
 use rustc_middle::ty::{
-    self, AdtDef, DefIdTree, ParamEnv, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable,
-    TypeVisitableExt,
+    self, AdtDef, ParamEnv, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
 };
 use rustc_session::lint::builtin::{UNINHABITED_STATIC, UNSUPPORTED_CALLING_CONVENTIONS};
 use rustc_span::symbol::sym;
 use rustc_span::{self, Span};
+use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::traits::error_reporting::on_unimplemented::OnUnimplementedDirective;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
@@ -115,9 +115,11 @@ fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: LocalDefId) -> b
                     allowed_union_field(*elem, tcx, param_env)
                 }
                 _ => {
-                    // Fallback case: allow `ManuallyDrop` and things that are `Copy`.
+                    // Fallback case: allow `ManuallyDrop` and things that are `Copy`,
+                    // also no need to report an error if the type is unresolved.
                     ty.ty_adt_def().is_some_and(|adt_def| adt_def.is_manually_drop())
                         || ty.is_copy_modulo_regions(tcx, param_env)
+                        || ty.references_error()
                 }
             }
         }
@@ -132,26 +134,14 @@ fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: LocalDefId) -> b
                     Some(Node::Field(field)) => (field.span, field.ty.span),
                     _ => unreachable!("mir field has to correspond to hir field"),
                 };
-                struct_span_err!(
-                    tcx.sess,
+                tcx.sess.emit_err(errors::InvalidUnionField {
                     field_span,
-                    E0740,
-                    "unions cannot contain fields that may need dropping"
-                )
-                .note(
-                    "a type is guaranteed not to need dropping \
-                    when it implements `Copy`, or when it is the special `ManuallyDrop<_>` type",
-                )
-                .multipart_suggestion_verbose(
-                    "when the type does not implement `Copy`, \
-                    wrap it inside a `ManuallyDrop<_>` and ensure it is manually dropped",
-                    vec![
-                        (ty_span.shrink_to_lo(), "std::mem::ManuallyDrop<".into()),
-                        (ty_span.shrink_to_hi(), ">".into()),
-                    ],
-                    Applicability::MaybeIncorrect,
-                )
-                .emit();
+                    sugg: errors::InvalidUnionFieldSuggestion {
+                        lo: ty_span.shrink_to_lo(),
+                        hi: ty_span.shrink_to_hi(),
+                    },
+                    note: (),
+                });
                 return false;
             } else if field_ty.needs_drop(tcx, param_env) {
                 // This should never happen. But we can get here e.g. in case of name resolution errors.
@@ -222,7 +212,7 @@ fn check_opaque(tcx: TyCtxt<'_>, id: hir::ItemId) {
         return;
     }
 
-    let substs = InternalSubsts::identity_for_item(tcx, item.owner_id.to_def_id());
+    let substs = InternalSubsts::identity_for_item(tcx, item.owner_id);
     let span = tcx.def_span(item.owner_id.def_id);
 
     if !tcx.features().impl_trait_projections {
@@ -315,8 +305,8 @@ pub(super) fn check_opaque_for_inheriting_lifetimes(
         ..
     }) = item.kind
     {
-        let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
-        let opaque_identity_ty = if in_trait {
+        let substs = InternalSubsts::identity_for_item(tcx, def_id);
+        let opaque_identity_ty = if in_trait && !tcx.lower_impl_trait_in_trait_to_assoc_ty() {
             tcx.mk_projection(def_id.to_def_id(), substs)
         } else {
             tcx.mk_opaque(def_id.to_def_id(), substs)
@@ -455,18 +445,15 @@ fn check_opaque_meets_bounds<'tcx>(
     // version.
     let errors = ocx.select_all_or_error();
     if !errors.is_empty() {
-        infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+        infcx.err_ctxt().report_fulfillment_errors(&errors);
     }
     match origin {
         // Checked when type checking the function containing them.
         hir::OpaqueTyOrigin::FnReturn(..) | hir::OpaqueTyOrigin::AsyncFn(..) => {}
         // Can have different predicates to their defining use
         hir::OpaqueTyOrigin::TyAlias => {
-            let outlives_environment = OutlivesEnvironment::new(param_env);
-            let _ = infcx.err_ctxt().check_region_obligations_and_report_errors(
-                defining_use_anchor,
-                &outlives_environment,
-            );
+            let outlives_env = OutlivesEnvironment::new(param_env);
+            let _ = ocx.resolve_regions_and_report_errors(defining_use_anchor, &outlives_env);
         }
     }
     // Clean up after ourselves
@@ -485,7 +472,7 @@ fn is_enum_of_nonnullable_ptr<'tcx>(
     let [var_one, var_two] = &adt_def.variants().raw[..] else {
         return false;
     };
-    let (([], [field]) | ([field], [])) = (&var_one.fields[..], &var_two.fields[..]) else {
+    let (([], [field]) | ([field], [])) = (&var_one.fields.raw[..], &var_two.fields.raw[..]) else {
         return false;
     };
     matches!(field.ty(tcx, substs).kind(), ty::FnPtr(..) | ty::Ref(..))
@@ -546,7 +533,7 @@ fn check_item_type(tcx: TyCtxt<'_>, id: hir::ItemId) {
                     }
                     ty::AssocKind::Type if assoc_item.defaultness(tcx).has_value() => {
                         let trait_substs =
-                            InternalSubsts::identity_for_item(tcx, id.owner_id.to_def_id());
+                            InternalSubsts::identity_for_item(tcx, id.owner_id);
                         let _: Result<_, rustc_errors::ErrorGuaranteed> = check_type_bounds(
                             tcx,
                             assoc_item,
@@ -565,10 +552,18 @@ fn check_item_type(tcx: TyCtxt<'_>, id: hir::ItemId) {
             check_union(tcx, id.owner_id.def_id);
         }
         DefKind::OpaqueTy => {
-            check_opaque(tcx, id);
+            let opaque = tcx.hir().expect_item(id.owner_id.def_id).expect_opaque_ty();
+            if let hir::OpaqueTyOrigin::FnReturn(fn_def_id) | hir::OpaqueTyOrigin::AsyncFn(fn_def_id) = opaque.origin
+                && let hir::Node::TraitItem(trait_item) = tcx.hir().get_by_def_id(fn_def_id)
+                && let (_, hir::TraitFn::Required(..)) = trait_item.expect_fn()
+            {
+                // Skip opaques from RPIT in traits with no default body.
+            } else {
+                check_opaque(tcx, id);
+            }
         }
         DefKind::ImplTraitPlaceholder => {
-            let parent = tcx.impl_trait_in_trait_parent(id.owner_id.to_def_id());
+            let parent = tcx.impl_trait_in_trait_parent_fn(id.owner_id.to_def_id());
             // Only check the validity of this opaque type if the function has a default body
             if let hir::Node::TraitItem(hir::TraitItem {
                 kind: hir::TraitItemKind::Fn(_, hir::TraitFn::Provided(_)),
@@ -896,7 +891,7 @@ pub fn check_simd(tcx: TyCtxt<'_>, sp: Span, def_id: LocalDefId) {
             struct_span_err!(tcx.sess, sp, E0075, "SIMD vector cannot be empty").emit();
             return;
         }
-        let e = fields[0].ty(tcx, substs);
+        let e = fields[FieldIdx::from_u32(0)].ty(tcx, substs);
         if !fields.iter().all(|f| f.ty(tcx, substs) == e) {
             struct_span_err!(tcx.sess, sp, E0076, "SIMD vector should be homogeneous")
                 .span_label(sp, "SIMD elements must have the same type")
@@ -1172,7 +1167,7 @@ fn check_enum(tcx: TyCtxt<'_>, def_id: LocalDefId) {
     def.destructor(tcx); // force the destructor to be evaluated
 
     if def.variants().is_empty() {
-        if let Some(attr) = tcx.get_attrs(def_id.to_def_id(), sym::repr).next() {
+        if let Some(attr) = tcx.get_attrs(def_id, sym::repr).next() {
             struct_span_err!(
                 tcx.sess,
                 attr.span,
@@ -1511,6 +1506,14 @@ fn opaque_type_cycle_error(
                     {
                         label_match(interior_ty.ty, interior_ty.span);
                     }
+                    if tcx.sess.opts.unstable_opts.drop_tracking_mir
+                        && let DefKind::Generator = tcx.def_kind(closure_def_id)
+                    {
+                        let generator_layout = tcx.mir_generator_witnesses(closure_def_id);
+                        for interior_ty in &generator_layout.field_tys {
+                            label_match(interior_ty.ty, interior_ty.source_info.span);
+                        }
+                    }
                 }
             }
         }
@@ -1548,6 +1551,6 @@ pub(super) fn check_generator_obligations(tcx: TyCtxt<'_>, def_id: LocalDefId) {
     let errors = fulfillment_cx.select_all_or_error(&infcx);
     debug!(?errors);
     if !errors.is_empty() {
-        infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+        infcx.err_ctxt().report_fulfillment_errors(&errors);
     }
 }
diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
index 691d3f8d9..5d119a773 100644
--- a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
+++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
@@ -16,8 +16,7 @@ use rustc_infer::traits::util;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::util::ExplicitSelf;
 use rustc_middle::ty::{
-    self, DefIdTree, InternalSubsts, Ty, TypeFoldable, TypeFolder, TypeSuperFoldable,
-    TypeVisitableExt,
+    self, InternalSubsts, Ty, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
 };
 use rustc_middle::ty::{GenericParamDefKind, ToPredicate, TyCtxt};
 use rustc_span::Span;
@@ -321,7 +320,7 @@ fn compare_method_predicate_entailment<'tcx>(
                 });
             }
             CheckImpliedWfMode::Skip => {
-                let reported = infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+                let reported = infcx.err_ctxt().report_fulfillment_errors(&errors);
                 return Err(reported);
             }
         }
@@ -331,13 +330,8 @@ fn compare_method_predicate_entailment<'tcx>(
     // lifetime parameters.
     let outlives_env = OutlivesEnvironment::with_bounds(
         param_env,
-        Some(infcx),
         infcx.implied_bounds_tys(param_env, impl_m_def_id, wf_tys.clone()),
     );
-    infcx.process_registered_region_obligations(
-        outlives_env.region_bound_pairs(),
-        outlives_env.param_env,
-    );
     let errors = infcx.resolve_regions(&outlives_env);
     if !errors.is_empty() {
         // FIXME(compiler-errors): This can be simplified when IMPLIED_BOUNDS_ENTAILMENT
@@ -584,13 +578,13 @@ fn compare_asyncness<'tcx>(
 #[instrument(skip(tcx), level = "debug", ret)]
 pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
     tcx: TyCtxt<'tcx>,
-    def_id: DefId,
+    impl_m_def_id: LocalDefId,
 ) -> Result<&'tcx FxHashMap<DefId, Ty<'tcx>>, ErrorGuaranteed> {
-    let impl_m = tcx.opt_associated_item(def_id).unwrap();
+    let impl_m = tcx.opt_associated_item(impl_m_def_id.to_def_id()).unwrap();
     let trait_m = tcx.opt_associated_item(impl_m.trait_item_def_id.unwrap()).unwrap();
     let impl_trait_ref =
         tcx.impl_trait_ref(impl_m.impl_container(tcx).unwrap()).unwrap().subst_identity();
-    let param_env = tcx.param_env(def_id);
+    let param_env = tcx.param_env(impl_m_def_id);
 
     // First, check a few of the same things as `compare_impl_method`,
     // just so we don't ICE during substitution later.
@@ -600,7 +594,6 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
 
     let trait_to_impl_substs = impl_trait_ref.substs;
 
-    let impl_m_def_id = impl_m.def_id.expect_local();
     let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m_def_id);
     let return_span = tcx.hir().fn_decl_by_hir_id(impl_m_hir_id).unwrap().output.span();
     let cause = ObligationCause::new(
@@ -721,23 +714,22 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
     // RPITs.
     let errors = ocx.select_all_or_error();
     if !errors.is_empty() {
-        let reported = infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+        let reported = infcx.err_ctxt().report_fulfillment_errors(&errors);
         return Err(reported);
     }
 
+    let collected_types = collector.types;
+
     // Finally, resolve all regions. This catches wily misuses of
     // lifetime parameters.
-    let outlives_environment = OutlivesEnvironment::with_bounds(
+    let outlives_env = OutlivesEnvironment::with_bounds(
         param_env,
-        Some(infcx),
         infcx.implied_bounds_tys(param_env, impl_m_def_id, wf_tys),
     );
-    infcx
-        .err_ctxt()
-        .check_region_obligations_and_report_errors(impl_m_def_id, &outlives_environment)?;
+    ocx.resolve_regions_and_report_errors(impl_m_def_id, &outlives_env)?;
 
     let mut collected_tys = FxHashMap::default();
-    for (def_id, (ty, substs)) in collector.types {
+    for (def_id, (ty, substs)) in collected_types {
         match infcx.fully_resolve(ty) {
             Ok(ty) => {
                 // `ty` contains free regions that we created earlier while liberating the
@@ -831,7 +823,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ImplTraitInTraitCollector<'_, 'tcx> {
 
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
         if let ty::Alias(ty::Projection, proj) = ty.kind()
-            && self.interner().def_kind(proj.def_id) == DefKind::ImplTraitPlaceholder
+            && self.interner().is_impl_trait_in_trait(proj.def_id)
         {
             if let Some((ty, _)) = self.types.get(&proj.def_id) {
                 return *ty;
@@ -1206,6 +1198,17 @@ fn compare_number_of_generics<'tcx>(
         return Ok(());
     }
 
+    // We never need to emit a separate error for RPITITs, since if an RPITIT
+    // has mismatched type or const generic arguments, then the method that it's
+    // inheriting the generics from will also have mismatched arguments, and
+    // we'll report an error for that instead. Delay a bug for safety, though.
+    if tcx.opt_rpitit_info(trait_.def_id).is_some() {
+        return Err(tcx.sess.delay_span_bug(
+            rustc_span::DUMMY_SP,
+            "errors comparing numbers of generics of trait/impl functions were not emitted",
+        ));
+    }
+
     let matchings = [
         ("type", trait_own_counts.types, impl_own_counts.types),
         ("const", trait_own_counts.consts, impl_own_counts.consts),
@@ -1732,14 +1735,11 @@ pub(super) fn compare_impl_const_raw(
     // version.
     let errors = ocx.select_all_or_error();
     if !errors.is_empty() {
-        return Err(infcx.err_ctxt().report_fulfillment_errors(&errors, None));
+        return Err(infcx.err_ctxt().report_fulfillment_errors(&errors));
     }
 
-    let outlives_environment = OutlivesEnvironment::new(param_env);
-    infcx
-        .err_ctxt()
-        .check_region_obligations_and_report_errors(impl_const_item_def, &outlives_environment)?;
-    Ok(())
+    let outlives_env = OutlivesEnvironment::new(param_env);
+    ocx.resolve_regions_and_report_errors(impl_const_item_def, &outlives_env)
 }
 
 pub(super) fn compare_impl_ty<'tcx>(
@@ -1832,19 +1832,14 @@ fn compare_type_predicate_entailment<'tcx>(
     // version.
     let errors = ocx.select_all_or_error();
     if !errors.is_empty() {
-        let reported = infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+        let reported = infcx.err_ctxt().report_fulfillment_errors(&errors);
         return Err(reported);
     }
 
     // Finally, resolve all regions. This catches wily misuses of
     // lifetime parameters.
-    let outlives_environment = OutlivesEnvironment::new(param_env);
-    infcx.err_ctxt().check_region_obligations_and_report_errors(
-        impl_ty.def_id.expect_local(),
-        &outlives_environment,
-    )?;
-
-    Ok(())
+    let outlives_env = OutlivesEnvironment::new(param_env);
+    ocx.resolve_regions_and_report_errors(impl_ty_def_id, &outlives_env)
 }
 
 /// Validate that `ProjectionCandidate`s created for this associated type will
@@ -1867,14 +1862,17 @@ pub(super) fn check_type_bounds<'tcx>(
     impl_ty: ty::AssocItem,
     impl_trait_ref: ty::TraitRef<'tcx>,
 ) -> Result<(), ErrorGuaranteed> {
+    let param_env = tcx.param_env(impl_ty.def_id);
+    let container_id = impl_ty.container_id(tcx);
     // Given
     //
     // impl<A, B> Foo<u32> for (A, B) {
-    //     type Bar<C> =...
+    //     type Bar<C> = Wrapper<A, B, C>
     // }
     //
     // - `impl_trait_ref` would be `<(A, B) as Foo<u32>>`
-    // - `impl_ty_substs` would be `[A, B, ^0.0]` (`^0.0` here is the bound var with db 0 and index 0)
+    // - `normalize_impl_ty_substs` would be `[A, B, ^0.0]` (`^0.0` here is the bound var with db 0 and index 0)
+    // - `normalize_impl_ty` would be `Wrapper<A, B, ^0.0>`
     // - `rebased_substs` would be `[(A, B), u32, ^0.0]`, combining the substs from
     //    the *trait* with the generic associated type parameters (as bound vars).
     //
@@ -1903,56 +1901,46 @@ pub(super) fn check_type_bounds<'tcx>(
     // Member<C: Eq> = .... That type would fail a well-formedness check that we ought to be doing
     // elsewhere, which would check that any <T as Family>::Member<X> meets the bounds declared in
     // the trait (notably, that X: Eq and T: Family).
-    let defs: &ty::Generics = tcx.generics_of(impl_ty.def_id);
-    let mut substs = smallvec::SmallVec::with_capacity(defs.count());
-    if let Some(def_id) = defs.parent {
-        let parent_defs = tcx.generics_of(def_id);
-        InternalSubsts::fill_item(&mut substs, tcx, parent_defs, &mut |param, _| {
-            tcx.mk_param_from_def(param)
-        });
-    }
     let mut bound_vars: smallvec::SmallVec<[ty::BoundVariableKind; 8]> =
-        smallvec::SmallVec::with_capacity(defs.count());
-    InternalSubsts::fill_single(&mut substs, defs, &mut |param, _| match param.kind {
-        GenericParamDefKind::Type { .. } => {
-            let kind = ty::BoundTyKind::Param(param.def_id, param.name);
-            let bound_var = ty::BoundVariableKind::Ty(kind);
-            bound_vars.push(bound_var);
-            tcx.mk_bound(
-                ty::INNERMOST,
-                ty::BoundTy { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind },
-            )
-            .into()
-        }
-        GenericParamDefKind::Lifetime => {
-            let kind = ty::BoundRegionKind::BrNamed(param.def_id, param.name);
-            let bound_var = ty::BoundVariableKind::Region(kind);
-            bound_vars.push(bound_var);
-            tcx.mk_re_late_bound(
-                ty::INNERMOST,
-                ty::BoundRegion { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind },
-            )
-            .into()
-        }
-        GenericParamDefKind::Const { .. } => {
-            let bound_var = ty::BoundVariableKind::Const;
-            bound_vars.push(bound_var);
-            tcx.mk_const(
-                ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_usize(bound_vars.len() - 1)),
-                tcx.type_of(param.def_id).subst_identity(),
-            )
-            .into()
-        }
-    });
-    let bound_vars = tcx.mk_bound_variable_kinds(&bound_vars);
-    let impl_ty_substs = tcx.mk_substs(&substs);
-    let container_id = impl_ty.container_id(tcx);
-
-    let rebased_substs = impl_ty_substs.rebase_onto(tcx, container_id, impl_trait_ref.substs);
-    let impl_ty_value = tcx.type_of(impl_ty.def_id).subst_identity();
-
-    let param_env = tcx.param_env(impl_ty.def_id);
-
+        smallvec::SmallVec::with_capacity(tcx.generics_of(impl_ty.def_id).params.len());
+    // Extend the impl's identity substs with late-bound GAT vars
+    let normalize_impl_ty_substs = ty::InternalSubsts::identity_for_item(tcx, container_id)
+        .extend_to(tcx, impl_ty.def_id, |param, _| match param.kind {
+            GenericParamDefKind::Type { .. } => {
+                let kind = ty::BoundTyKind::Param(param.def_id, param.name);
+                let bound_var = ty::BoundVariableKind::Ty(kind);
+                bound_vars.push(bound_var);
+                tcx.mk_bound(
+                    ty::INNERMOST,
+                    ty::BoundTy { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind },
+                )
+                .into()
+            }
+            GenericParamDefKind::Lifetime => {
+                let kind = ty::BoundRegionKind::BrNamed(param.def_id, param.name);
+                let bound_var = ty::BoundVariableKind::Region(kind);
+                bound_vars.push(bound_var);
+                tcx.mk_re_late_bound(
+                    ty::INNERMOST,
+                    ty::BoundRegion { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind },
+                )
+                .into()
+            }
+            GenericParamDefKind::Const { .. } => {
+                let bound_var = ty::BoundVariableKind::Const;
+                bound_vars.push(bound_var);
+                tcx.mk_const(
+                    ty::ConstKind::Bound(
+                        ty::INNERMOST,
+                        ty::BoundVar::from_usize(bound_vars.len() - 1),
+                    ),
+                    tcx.type_of(param.def_id)
+                        .no_bound_vars()
+                        .expect("const parameter types cannot be generic"),
+                )
+                .into()
+            }
+        });
     // When checking something like
     //
     // trait X { type Y: PartialEq<<Self as X>::Y> }
@@ -1962,9 +1950,13 @@ pub(super) fn check_type_bounds<'tcx>(
     // we want <T as X>::Y to normalize to S. This is valid because we are
     // checking the default value specifically here. Add this equality to the
     // ParamEnv for normalization specifically.
+    let normalize_impl_ty = tcx.type_of(impl_ty.def_id).subst(tcx, normalize_impl_ty_substs);
+    let rebased_substs =
+        normalize_impl_ty_substs.rebase_onto(tcx, container_id, impl_trait_ref.substs);
+    let bound_vars = tcx.mk_bound_variable_kinds(&bound_vars);
     let normalize_param_env = {
         let mut predicates = param_env.caller_bounds().iter().collect::<Vec<_>>();
-        match impl_ty_value.kind() {
+        match normalize_impl_ty.kind() {
             ty::Alias(ty::Projection, proj)
                 if proj.def_id == trait_ty.def_id && proj.substs == rebased_substs =>
             {
@@ -1978,7 +1970,7 @@ pub(super) fn check_type_bounds<'tcx>(
                 ty::Binder::bind_with_vars(
                     ty::ProjectionPredicate {
                         projection_ty: tcx.mk_alias_ty(trait_ty.def_id, rebased_substs),
-                        term: impl_ty_value.into(),
+                        term: normalize_impl_ty.into(),
                     },
                     bound_vars,
                 )
@@ -1996,13 +1988,20 @@ pub(super) fn check_type_bounds<'tcx>(
     let infcx = tcx.infer_ctxt().build();
     let ocx = ObligationCtxt::new(&infcx);
 
-    let impl_ty_span = match tcx.hir().get_by_def_id(impl_ty_def_id) {
-        hir::Node::TraitItem(hir::TraitItem {
-            kind: hir::TraitItemKind::Type(_, Some(ty)),
-            ..
-        }) => ty.span,
-        hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Type(ty), .. }) => ty.span,
-        _ => bug!(),
+    // A synthetic impl Trait for RPITIT desugaring has no HIR, which we currently use to get the
+    // span for an impl's associated type. Instead, for these, use the def_span for the synthesized
+    // associated type.
+    let impl_ty_span = if tcx.opt_rpitit_info(impl_ty.def_id).is_some() {
+        tcx.def_span(impl_ty_def_id)
+    } else {
+        match tcx.hir().get_by_def_id(impl_ty_def_id) {
+            hir::Node::TraitItem(hir::TraitItem {
+                kind: hir::TraitItemKind::Type(_, Some(ty)),
+                ..
+            }) => ty.span,
+            hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Type(ty), .. }) => ty.span,
+            _ => bug!(),
+        }
     };
     let assumed_wf_types = ocx.assumed_wf_types(param_env, impl_ty_span, impl_ty_def_id);
 
@@ -2023,7 +2022,7 @@ pub(super) fn check_type_bounds<'tcx>(
         ObligationCause::new(impl_ty_span, impl_ty_def_id, code)
     };
 
-    let obligations = tcx
+    let obligations: Vec<_> = tcx
         .bound_explicit_item_bounds(trait_ty.def_id)
         .subst_iter_copied(tcx, rebased_substs)
         .map(|(concrete_ty_bound, span)| {
@@ -2033,7 +2032,7 @@ pub(super) fn check_type_bounds<'tcx>(
         .collect();
     debug!("check_type_bounds: item_bounds={:?}", obligations);
 
-    for mut obligation in util::elaborate_obligations(tcx, obligations) {
+    for mut obligation in util::elaborate(tcx, obligations) {
         let normalized_predicate =
             ocx.normalize(&normalize_cause, normalize_param_env, obligation.predicate);
         debug!("compare_projection_bounds: normalized predicate = {:?}", normalized_predicate);
@@ -2045,22 +2044,15 @@ pub(super) fn check_type_bounds<'tcx>(
     // version.
     let errors = ocx.select_all_or_error();
     if !errors.is_empty() {
-        let reported = infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+        let reported = infcx.err_ctxt().report_fulfillment_errors(&errors);
         return Err(reported);
     }
 
     // Finally, resolve all regions. This catches wily misuses of
     // lifetime parameters.
     let implied_bounds = infcx.implied_bounds_tys(param_env, impl_ty_def_id, assumed_wf_types);
-    let outlives_environment =
-        OutlivesEnvironment::with_bounds(param_env, Some(&infcx), implied_bounds);
-
-    infcx.err_ctxt().check_region_obligations_and_report_errors(
-        impl_ty.def_id.expect_local(),
-        &outlives_environment,
-    )?;
-
-    Ok(())
+    let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds);
+    ocx.resolve_regions_and_report_errors(impl_ty_def_id, &outlives_env)
 }
 
 fn assoc_item_kind_str(impl_item: &ty::AssocItem) -> &'static str {
diff --git a/compiler/rustc_hir_analysis/src/check/dropck.rs b/compiler/rustc_hir_analysis/src/check/dropck.rs
index 2bb724138..111bf5e54 100644
--- a/compiler/rustc_hir_analysis/src/check/dropck.rs
+++ b/compiler/rustc_hir_analysis/src/check/dropck.rs
@@ -253,10 +253,6 @@ impl<'tcx> TypeRelation<'tcx> for SimpleEqRelation<'tcx> {
         self.tcx
     }
 
-    fn intercrate(&self) -> bool {
-        false
-    }
-
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
         self.param_env
     }
@@ -269,10 +265,6 @@ impl<'tcx> TypeRelation<'tcx> for SimpleEqRelation<'tcx> {
         true
     }
 
-    fn mark_ambiguous(&mut self) {
-        bug!()
-    }
-
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
         _: ty::Variance,
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
index 054284cce..854974d16 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
@@ -59,6 +59,7 @@ fn equate_intrinsic_type<'tcx>(
         require_same_types(
             tcx,
             &cause,
+            ty::ParamEnv::empty(), // FIXME: do all intrinsics have an empty param env?
             tcx.mk_fn_ptr(tcx.fn_sig(it.owner_id).subst_identity()),
             fty,
         );
@@ -138,14 +139,14 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
     let name_str = intrinsic_name.as_str();
 
     let bound_vars = tcx.mk_bound_variable_kinds(&[
-        ty::BoundVariableKind::Region(ty::BrAnon(0, None)),
+        ty::BoundVariableKind::Region(ty::BrAnon(None)),
         ty::BoundVariableKind::Region(ty::BrEnv),
     ]);
     let mk_va_list_ty = |mutbl| {
         tcx.lang_items().va_list().map(|did| {
             let region = tcx.mk_re_late_bound(
                 ty::INNERMOST,
-                ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(0, None) },
+                ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(None) },
             );
             let env_region = tcx.mk_re_late_bound(
                 ty::INNERMOST,
@@ -222,6 +223,21 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                 ],
                 tcx.mk_ptr(ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }),
             ),
+            sym::option_payload_ptr => {
+                let option_def_id = tcx.require_lang_item(hir::LangItem::Option, None);
+                let p0 = param(0);
+                (
+                    1,
+                    vec![tcx.mk_ptr(ty::TypeAndMut {
+                        ty: tcx.mk_adt(
+                            tcx.adt_def(option_def_id),
+                            tcx.mk_substs_from_iter([ty::GenericArg::from(p0)].into_iter()),
+                        ),
+                        mutbl: hir::Mutability::Not,
+                    })],
+                    tcx.mk_ptr(ty::TypeAndMut { ty: p0, mutbl: hir::Mutability::Not }),
+                )
+            }
             sym::ptr_mask => (
                 1,
                 vec![
@@ -300,6 +316,8 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             sym::nearbyintf64 => (0, vec![tcx.types.f64], tcx.types.f64),
             sym::roundf32 => (0, vec![tcx.types.f32], tcx.types.f32),
             sym::roundf64 => (0, vec![tcx.types.f64], tcx.types.f64),
+            sym::roundevenf32 => (0, vec![tcx.types.f32], tcx.types.f32),
+            sym::roundevenf64 => (0, vec![tcx.types.f64], tcx.types.f64),
 
             sym::volatile_load | sym::unaligned_volatile_load => {
                 (1, vec![tcx.mk_imm_ptr(param(0))], param(0))
@@ -361,14 +379,15 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             sym::likely => (0, vec![tcx.types.bool], tcx.types.bool),
             sym::unlikely => (0, vec![tcx.types.bool], tcx.types.bool),
 
+            sym::read_via_copy => (1, vec![tcx.mk_imm_ptr(param(0))], param(0)),
+
             sym::discriminant_value => {
                 let assoc_items = tcx.associated_item_def_ids(
                     tcx.require_lang_item(hir::LangItem::DiscriminantKind, None),
                 );
                 let discriminant_def_id = assoc_items[0];
 
-                let br =
-                    ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(0, None) };
+                let br = ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(None) };
                 (
                     1,
                     vec![tcx.mk_imm_ref(tcx.mk_re_late_bound(ty::INNERMOST, br), param(0))],
@@ -420,8 +439,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             sym::nontemporal_store => (1, vec![tcx.mk_mut_ptr(param(0)), param(0)], tcx.mk_unit()),
 
             sym::raw_eq => {
-                let br =
-                    ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(0, None) };
+                let br = ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(None) };
                 let param_ty = tcx.mk_imm_ref(tcx.mk_re_late_bound(ty::INNERMOST, br), param(0));
                 (1, vec![param_ty; 2], tcx.types.bool)
             }
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsicck.rs b/compiler/rustc_hir_analysis/src/check/intrinsicck.rs
index b1d5a27be..0d482b53a 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsicck.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsicck.rs
@@ -1,10 +1,11 @@
 use rustc_ast::InlineAsmTemplatePiece;
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_hir as hir;
 use rustc_middle::ty::{self, Article, FloatTy, IntTy, Ty, TyCtxt, TypeVisitableExt, UintTy};
 use rustc_session::lint;
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{Symbol, DUMMY_SP};
+use rustc_target::abi::FieldIdx;
 use rustc_target::asm::{InlineAsmReg, InlineAsmRegClass, InlineAsmRegOrRegClass, InlineAsmType};
 
 pub struct InlineAsmCtxt<'a, 'tcx> {
@@ -51,7 +52,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
         template: &[InlineAsmTemplatePiece],
         is_input: bool,
         tied_input: Option<(&'tcx hir::Expr<'tcx>, Option<InlineAsmType>)>,
-        target_features: &FxHashSet<Symbol>,
+        target_features: &FxIndexSet<Symbol>,
     ) -> Option<InlineAsmType> {
         let ty = (self.get_operand_ty)(expr);
         if ty.has_non_region_infer() {
@@ -82,7 +83,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
             }
             ty::Adt(adt, substs) if adt.repr().simd() => {
                 let fields = &adt.non_enum_variant().fields;
-                let elem_ty = fields[0].ty(self.tcx, substs);
+                let elem_ty = fields[FieldIdx::from_u32(0)].ty(self.tcx, substs);
                 match elem_ty.kind() {
                     ty::Never | ty::Error(_) => return None,
                     ty::Int(IntTy::I8) | ty::Uint(UintTy::U8) => {
@@ -201,7 +202,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
         // (!). In that case we still need the earlier check to verify that the
         // register class is usable at all.
         if let Some(feature) = feature {
-            if !target_features.contains(&feature) {
+            if !target_features.contains(feature) {
                 let msg = &format!("`{}` target feature is not enabled", feature);
                 let mut err = self.tcx.sess.struct_span_err(expr.span, msg);
                 err.note(&format!(
diff --git a/compiler/rustc_hir_analysis/src/check/mod.rs b/compiler/rustc_hir_analysis/src/check/mod.rs
index 9acfc1b3d..8fe4c44fc 100644
--- a/compiler/rustc_hir_analysis/src/check/mod.rs
+++ b/compiler/rustc_hir_analysis/src/check/mod.rs
@@ -109,8 +109,8 @@ pub fn provide(providers: &mut Providers) {
     };
 }
 
-fn adt_destructor(tcx: TyCtxt<'_>, def_id: DefId) -> Option<ty::Destructor> {
-    tcx.calculate_dtor(def_id, dropck::check_drop_impl)
+fn adt_destructor(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<ty::Destructor> {
+    tcx.calculate_dtor(def_id.to_def_id(), dropck::check_drop_impl)
 }
 
 /// Given a `DefId` for an opaque type in return position, find its parent item's return
@@ -202,8 +202,11 @@ fn missing_items_err(
     missing_items: &[ty::AssocItem],
     full_impl_span: Span,
 ) {
+    let missing_items =
+        missing_items.iter().filter(|trait_item| tcx.opt_rpitit_info(trait_item.def_id).is_none());
+
     let missing_items_msg = missing_items
-        .iter()
+        .clone()
         .map(|trait_item| trait_item.name.to_string())
         .collect::<Vec<_>>()
         .join("`, `");
diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
index 4cccdf30c..53197bc84 100644
--- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs
+++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
@@ -1,7 +1,6 @@
 use crate::autoderef::Autoderef;
 use crate::constrained_generic_params::{identify_constrained_generic_params, Parameter};
 
-use hir::def::DefKind;
 use rustc_ast as ast;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
 use rustc_errors::{pluralize, struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed};
@@ -111,16 +110,13 @@ pub(super) fn enter_wf_checking_ctxt<'tcx, F>(
 
     let errors = wfcx.select_all_or_error();
     if !errors.is_empty() {
-        infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+        infcx.err_ctxt().report_fulfillment_errors(&errors);
         return;
     }
 
-    let outlives_environment =
-        OutlivesEnvironment::with_bounds(param_env, Some(infcx), implied_bounds);
+    let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds);
 
-    let _ = infcx
-        .err_ctxt()
-        .check_region_obligations_and_report_errors(body_def_id, &outlives_environment);
+    let _ = wfcx.ocx.resolve_regions_and_report_errors(body_def_id, &outlives_env);
 }
 
 fn check_well_formed(tcx: TyCtxt<'_>, def_id: hir::OwnerId) {
@@ -676,19 +672,13 @@ fn resolve_regions_with_wf_tys<'tcx>(
     let infcx = tcx.infer_ctxt().build();
     let outlives_environment = OutlivesEnvironment::with_bounds(
         param_env,
-        Some(&infcx),
         infcx.implied_bounds_tys(param_env, id, wf_tys.clone()),
     );
     let region_bound_pairs = outlives_environment.region_bound_pairs();
 
     add_constraints(&infcx, region_bound_pairs);
 
-    infcx.process_registered_region_obligations(
-        outlives_environment.region_bound_pairs(),
-        param_env,
-    );
     let errors = infcx.resolve_regions(&outlives_environment);
-
     debug!(?errors, "errors");
 
     // If we were able to prove that the type outlives the region without
@@ -1033,7 +1023,7 @@ fn check_type_defn<'tcx>(tcx: TyCtxt<'tcx>, item: &hir::Item<'tcx>, all_sized: b
             // intermediate types must be sized.
             let needs_drop_copy = || {
                 packed && {
-                    let ty = tcx.type_of(variant.fields.last().unwrap().did).subst_identity();
+                    let ty = tcx.type_of(variant.fields.raw.last().unwrap().did).subst_identity();
                     let ty = tcx.erase_regions(ty);
                     if ty.needs_infer() {
                         tcx.sess
@@ -1049,7 +1039,7 @@ fn check_type_defn<'tcx>(tcx: TyCtxt<'tcx>, item: &hir::Item<'tcx>, all_sized: b
             let all_sized = all_sized || variant.fields.is_empty() || needs_drop_copy();
             let unsized_len = if all_sized { 0 } else { 1 };
             for (idx, field) in
-                variant.fields[..variant.fields.len() - unsized_len].iter().enumerate()
+                variant.fields.raw[..variant.fields.len() - unsized_len].iter().enumerate()
             {
                 let last = idx == variant.fields.len() - 1;
                 let field_id = field.did.expect_local();
@@ -1545,31 +1535,81 @@ fn check_return_position_impl_trait_in_trait_bounds<'tcx>(
     span: Span,
 ) {
     let tcx = wfcx.tcx();
-    if let Some(assoc_item) = tcx.opt_associated_item(fn_def_id.to_def_id())
-        && assoc_item.container == ty::AssocItemContainer::TraitContainer
-    {
-        for arg in fn_output.walk() {
-            if let ty::GenericArgKind::Type(ty) = arg.unpack()
-                && let ty::Alias(ty::Opaque, proj) = ty.kind()
-                && tcx.def_kind(proj.def_id) == DefKind::ImplTraitPlaceholder
-                && tcx.impl_trait_in_trait_parent(proj.def_id) == fn_def_id.to_def_id()
+    let Some(assoc_item) = tcx.opt_associated_item(fn_def_id.to_def_id()) else {
+        return;
+    };
+    if assoc_item.container != ty::AssocItemContainer::TraitContainer {
+        return;
+    }
+    fn_output.visit_with(&mut ImplTraitInTraitFinder {
+        wfcx,
+        fn_def_id,
+        depth: ty::INNERMOST,
+        seen: FxHashSet::default(),
+    });
+}
+
+// FIXME(-Zlower-impl-trait-in-trait-to-assoc-ty): Even with the new lowering
+// strategy, we can't just call `check_associated_item` on the new RPITITs,
+// because tests like `tests/ui/async-await/in-trait/implied-bounds.rs` will fail.
+// That's because we need to check that the bounds of the RPITIT hold using
+// the special substs that we create during opaque type lowering, otherwise we're
+// getting a bunch of early bound and free regions mixed up... Haven't looked too
+// deep into this, though.
+struct ImplTraitInTraitFinder<'a, 'tcx> {
+    wfcx: &'a WfCheckingCtxt<'a, 'tcx>,
+    fn_def_id: LocalDefId,
+    depth: ty::DebruijnIndex,
+    seen: FxHashSet<DefId>,
+}
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ImplTraitInTraitFinder<'_, 'tcx> {
+    type BreakTy = !;
+
+    fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<!> {
+        let tcx = self.wfcx.tcx();
+        if let ty::Alias(ty::Opaque, unshifted_opaque_ty) = *ty.kind()
+            && self.seen.insert(unshifted_opaque_ty.def_id)
+            && let Some(opaque_def_id) = unshifted_opaque_ty.def_id.as_local()
+            && let opaque = tcx.hir().expect_item(opaque_def_id).expect_opaque_ty()
+            && let hir::OpaqueTyOrigin::FnReturn(source) | hir::OpaqueTyOrigin::AsyncFn(source) = opaque.origin
+            && source == self.fn_def_id
+        {
+            let opaque_ty = tcx.fold_regions(unshifted_opaque_ty, |re, depth| {
+                if let ty::ReLateBound(index, bv) = re.kind() {
+                    if depth != ty::INNERMOST {
+                        return tcx.mk_re_error_with_message(
+                            DUMMY_SP,
+                            "we shouldn't walk non-predicate binders with `impl Trait`...",
+                        );
+                    }
+                    tcx.mk_re_late_bound(index.shifted_out_to_binder(self.depth), bv)
+                } else {
+                    re
+                }
+            });
+            for (bound, bound_span) in tcx
+                .bound_explicit_item_bounds(opaque_ty.def_id)
+                .subst_iter_copied(tcx, opaque_ty.substs)
             {
-                let span = tcx.def_span(proj.def_id);
-                let bounds = wfcx.tcx().explicit_item_bounds(proj.def_id);
-                let wf_obligations = bounds.iter().flat_map(|&(bound, bound_span)| {
-                    let bound = ty::EarlyBinder(bound).subst(tcx, proj.substs);
-                    let normalized_bound = wfcx.normalize(span, None, bound);
-                    traits::wf::predicate_obligations(
-                        wfcx.infcx,
-                        wfcx.param_env,
-                        wfcx.body_def_id,
-                        normalized_bound,
-                        bound_span,
-                    )
-                });
-                wfcx.register_obligations(wf_obligations);
+                let bound = self.wfcx.normalize(bound_span, None, bound);
+                self.wfcx.register_obligations(traits::wf::predicate_obligations(
+                    self.wfcx.infcx,
+                    self.wfcx.param_env,
+                    self.wfcx.body_def_id,
+                    bound,
+                    bound_span,
+                ));
+                // Set the debruijn index back to innermost here, since we already eagerly
+                // shifted the substs that we use to generate these bounds. This is unfortunately
+                // subtly different behavior than the `ImplTraitInTraitFinder` we use in `param_env`,
+                // but that function doesn't actually need to normalize the bound it's visiting
+                // (whereas we have to do so here)...
+                let old_depth = std::mem::replace(&mut self.depth, ty::INNERMOST);
+                bound.visit_with(self);
+                self.depth = old_depth;
             }
         }
+        ty.super_visit_with(self)
     }
 }
 
@@ -1784,7 +1824,7 @@ fn check_variances_for_type_defn<'tcx>(
 
     // Lazily calculated because it is only needed in case of an error.
     let explicitly_bounded_params = LazyCell::new(|| {
-        let icx = crate::collect::ItemCtxt::new(tcx, item.owner_id.to_def_id());
+        let icx = crate::collect::ItemCtxt::new(tcx, item.owner_id.def_id);
         hir_generics
             .predicates
             .iter()
@@ -1861,16 +1901,15 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
 
         let predicates_with_span = tcx.predicates_of(self.body_def_id).predicates.iter().copied();
         // Check elaborated bounds.
-        let implied_obligations = traits::elaborate_predicates_with_span(tcx, predicates_with_span);
+        let implied_obligations = traits::elaborate(tcx, predicates_with_span);
 
-        for obligation in implied_obligations {
+        for (pred, obligation_span) in implied_obligations {
             // We lower empty bounds like `Vec<dyn Copy>:` as
             // `WellFormed(Vec<dyn Copy>)`, which will later get checked by
             // regular WF checking
-            if let ty::PredicateKind::WellFormed(..) = obligation.predicate.kind().skip_binder() {
+            if let ty::PredicateKind::WellFormed(..) = pred.kind().skip_binder() {
                 continue;
             }
-            let pred = obligation.predicate;
             // Match the existing behavior.
             if pred.is_global() && !pred.has_late_bound_vars() {
                 let pred = self.normalize(span, None, pred);
@@ -1881,8 +1920,6 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
                 if let Some(hir::Generics { predicates, .. }) =
                     hir_node.and_then(|node| node.generics())
                 {
-                    let obligation_span = obligation.cause.span();
-
                     span = predicates
                         .iter()
                         // There seems to be no better way to find out which predicate we are in
diff --git a/compiler/rustc_hir_analysis/src/check_unused.rs b/compiler/rustc_hir_analysis/src/check_unused.rs
index f3f5851d8..268b9ac53 100644
--- a/compiler/rustc_hir_analysis/src/check_unused.rs
+++ b/compiler/rustc_hir_analysis/src/check_unused.rs
@@ -10,7 +10,7 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
     for item_def_id in tcx.hir().body_owners() {
         let imports = tcx.used_trait_imports(item_def_id);
         debug!("GatherVisitor: item_def_id={:?} with imports {:#?}", item_def_id, imports);
-        used_trait_imports.extend(imports.items().copied());
+        used_trait_imports.extend_unord(imports.items().copied());
     }
 
     for &id in tcx.maybe_unused_trait_imports(()) {
diff --git a/compiler/rustc_hir_analysis/src/coherence/builtin.rs b/compiler/rustc_hir_analysis/src/coherence/builtin.rs
index ffb68abf9..0f40cca94 100644
--- a/compiler/rustc_hir_analysis/src/coherence/builtin.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/builtin.rs
@@ -2,21 +2,23 @@
 //! up data structures required by type-checking/codegen.
 
 use crate::errors::{CopyImplOnNonAdt, CopyImplOnTypeWithDtor, DropImplOnWrongItem};
+use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{struct_span_err, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::ItemKind;
 use rustc_infer::infer::outlives::env::OutlivesEnvironment;
-use rustc_infer::infer::TyCtxtInferExt;
 use rustc_infer::infer::{self, RegionResolutionError};
+use rustc_infer::infer::{DefineOpaqueTypes, TyCtxtInferExt};
+use rustc_infer::traits::Obligation;
 use rustc_middle::ty::adjustment::CoerceUnsizedInfo;
 use rustc_middle::ty::{self, suggest_constraining_type_params, Ty, TyCtxt, TypeVisitableExt};
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt;
 use rustc_trait_selection::traits::misc::{
     type_allowed_to_implement_copy, CopyImplementationError, InfringingFieldsReason,
 };
-use rustc_trait_selection::traits::predicate_for_trait_def;
+use rustc_trait_selection::traits::ObligationCtxt;
 use rustc_trait_selection::traits::{self, ObligationCause};
 use std::collections::BTreeMap;
 
@@ -86,7 +88,7 @@ fn visit_implementation_of_copy(tcx: TyCtxt<'_>, impl_did: LocalDefId) {
                 tcx.sess,
                 span,
                 E0204,
-                "the trait `Copy` may not be implemented for this type"
+                "the trait `Copy` cannot be implemented for this type"
             );
 
             // We'll try to suggest constraining type parameters to fulfill the requirements of
@@ -94,7 +96,14 @@ fn visit_implementation_of_copy(tcx: TyCtxt<'_>, impl_did: LocalDefId) {
             let mut errors: BTreeMap<_, Vec<_>> = Default::default();
             let mut bounds = vec![];
 
+            let mut seen_tys = FxHashSet::default();
+
             for (field, ty, reason) in fields {
+                // Only report an error once per type.
+                if !seen_tys.insert(ty) {
+                    continue;
+                }
+
                 let field_span = tcx.def_span(field.did);
                 err.span_label(field_span, "this field does not implement `Copy`");
 
@@ -227,7 +236,8 @@ fn visit_implementation_of_dispatch_from_dyn(tcx: TyCtxt<'_>, impl_did: LocalDef
     use rustc_type_ir::sty::TyKind::*;
     match (source.kind(), target.kind()) {
         (&Ref(r_a, _, mutbl_a), Ref(r_b, _, mutbl_b))
-            if infcx.at(&cause, param_env).eq(r_a, *r_b).is_ok() && mutbl_a == *mutbl_b => {}
+            if infcx.at(&cause, param_env).eq(DefineOpaqueTypes::No, r_a, *r_b).is_ok()
+                && mutbl_a == *mutbl_b => {}
         (&RawPtr(tm_a), &RawPtr(tm_b)) if tm_a.mutbl == tm_b.mutbl => (),
         (&Adt(def_a, substs_a), &Adt(def_b, substs_b))
             if def_a.is_struct() && def_b.is_struct() =>
@@ -270,7 +280,9 @@ fn visit_implementation_of_dispatch_from_dyn(tcx: TyCtxt<'_>, impl_did: LocalDef
                         }
                     }
 
-                    if let Ok(ok) = infcx.at(&cause, param_env).eq(ty_a, ty_b) {
+                    if let Ok(ok) =
+                        infcx.at(&cause, param_env).eq(DefineOpaqueTypes::No, ty_a, ty_b)
+                    {
                         if ok.obligations.is_empty() {
                             create_err(
                                 "the trait `DispatchFromDyn` may only be implemented \
@@ -323,28 +335,26 @@ fn visit_implementation_of_dispatch_from_dyn(tcx: TyCtxt<'_>, impl_did: LocalDef
                     ))
                     .emit();
             } else {
-                let errors = traits::fully_solve_obligations(
-                    &infcx,
-                    coerced_fields.into_iter().map(|field| {
-                        predicate_for_trait_def(
-                            tcx,
-                            param_env,
-                            cause.clone(),
+                let ocx = ObligationCtxt::new(&infcx);
+                for field in coerced_fields {
+                    ocx.register_obligation(Obligation::new(
+                        tcx,
+                        cause.clone(),
+                        param_env,
+                        ty::Binder::dummy(tcx.mk_trait_ref(
                             dispatch_from_dyn_trait,
-                            0,
                             [field.ty(tcx, substs_a), field.ty(tcx, substs_b)],
-                        )
-                    }),
-                );
+                        )),
+                    ));
+                }
+                let errors = ocx.select_all_or_error();
                 if !errors.is_empty() {
-                    infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+                    infcx.err_ctxt().report_fulfillment_errors(&errors);
                 }
 
                 // Finally, resolve all regions.
                 let outlives_env = OutlivesEnvironment::new(param_env);
-                let _ = infcx
-                    .err_ctxt()
-                    .check_region_obligations_and_report_errors(impl_did, &outlives_env);
+                let _ = ocx.resolve_regions_and_report_errors(impl_did, &outlives_env);
             }
         }
         _ => {
@@ -357,11 +367,8 @@ fn visit_implementation_of_dispatch_from_dyn(tcx: TyCtxt<'_>, impl_did: LocalDef
     }
 }
 
-pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUnsizedInfo {
+pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: LocalDefId) -> CoerceUnsizedInfo {
     debug!("compute_coerce_unsized_info(impl_did={:?})", impl_did);
-
-    // this provider should only get invoked for local def-ids
-    let impl_did = impl_did.expect_local();
     let span = tcx.def_span(impl_did);
 
     let coerce_unsized_trait = tcx.require_lang_item(LangItem::CoerceUnsized, Some(span));
@@ -477,8 +484,7 @@ pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUn
             // U` can be coerced to `*mut V` if `U: Unsize<V>`.
             let fields = &def_a.non_enum_variant().fields;
             let diff_fields = fields
-                .iter()
-                .enumerate()
+                .iter_enumerated()
                 .filter_map(|(i, f)| {
                     let (a, b) = (f.ty(tcx, substs_a), f.ty(tcx, substs_b));
 
@@ -496,7 +502,7 @@ pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUn
                     // we may have to evaluate constraint
                     // expressions in the course of execution.)
                     // See e.g., #41936.
-                    if let Ok(ok) = infcx.at(&cause, param_env).eq(a, b) {
+                    if let Ok(ok) = infcx.at(&cause, param_env).eq(DefineOpaqueTypes::No, a, b) {
                         if ok.obligations.is_empty() {
                             return None;
                         }
@@ -572,17 +578,19 @@ pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUn
     };
 
     // Register an obligation for `A: Trait<B>`.
+    let ocx = ObligationCtxt::new(&infcx);
     let cause = traits::ObligationCause::misc(span, impl_did);
-    let predicate =
-        predicate_for_trait_def(tcx, param_env, cause, trait_def_id, 0, [source, target]);
-    let errors = traits::fully_solve_obligation(&infcx, predicate);
+    let obligation =
+        Obligation::new(tcx, cause, param_env, tcx.mk_trait_ref(trait_def_id, [source, target]));
+    ocx.register_obligation(obligation);
+    let errors = ocx.select_all_or_error();
     if !errors.is_empty() {
-        infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+        infcx.err_ctxt().report_fulfillment_errors(&errors);
     }
 
     // Finally, resolve all regions.
     let outlives_env = OutlivesEnvironment::new(param_env);
-    let _ = infcx.err_ctxt().check_region_obligations_and_report_errors(impl_did, &outlives_env);
+    let _ = ocx.resolve_regions_and_report_errors(impl_did, &outlives_env);
 
     CoerceUnsizedInfo { custom_kind: kind }
 }
diff --git a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
index 02f3eeee0..3d37e0ce0 100644
--- a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
@@ -10,7 +10,7 @@
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
-use rustc_hir::def_id::{CrateNum, DefId, LocalDefId};
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::ty::fast_reject::{simplify_type, SimplifiedType, TreatParams};
 use rustc_middle::ty::{self, CrateInherentImpls, Ty, TyCtxt};
 use rustc_span::symbol::sym;
@@ -24,7 +24,7 @@ pub fn crate_inherent_impls(tcx: TyCtxt<'_>, (): ()) -> CrateInherentImpls {
     collect.impls_map
 }
 
-pub fn crate_incoherent_impls(tcx: TyCtxt<'_>, (_, simp): (CrateNum, SimplifiedType)) -> &[DefId] {
+pub fn crate_incoherent_impls(tcx: TyCtxt<'_>, simp: SimplifiedType) -> &[DefId] {
     let crate_map = tcx.crate_inherent_impls(());
     tcx.arena.alloc_from_iter(
         crate_map.incoherent_impls.get(&simp).unwrap_or(&Vec::new()).iter().map(|d| d.to_def_id()),
@@ -32,9 +32,7 @@ pub fn crate_incoherent_impls(tcx: TyCtxt<'_>, (_, simp): (CrateNum, SimplifiedT
 }
 
 /// On-demand query: yields a vector of the inherent impls for a specific type.
-pub fn inherent_impls(tcx: TyCtxt<'_>, ty_def_id: DefId) -> &[DefId] {
-    let ty_def_id = ty_def_id.expect_local();
-
+pub fn inherent_impls(tcx: TyCtxt<'_>, ty_def_id: LocalDefId) -> &[DefId] {
     let crate_map = tcx.crate_inherent_impls(());
     match crate_map.inherent_impls.get(&ty_def_id) {
         Some(v) => &v[..],
@@ -99,7 +97,7 @@ impl<'tcx> InherentCollect<'tcx> {
                 }
             }
 
-            if let Some(simp) = simplify_type(self.tcx, self_ty, TreatParams::AsInfer) {
+            if let Some(simp) = simplify_type(self.tcx, self_ty, TreatParams::AsCandidateKey) {
                 self.impls_map.incoherent_impls.entry(simp).or_default().push(impl_def_id);
             } else {
                 bug!("unexpected self type: {:?}", self_ty);
@@ -159,7 +157,7 @@ impl<'tcx> InherentCollect<'tcx> {
             }
         }
 
-        if let Some(simp) = simplify_type(self.tcx, ty, TreatParams::AsInfer) {
+        if let Some(simp) = simplify_type(self.tcx, ty, TreatParams::AsCandidateKey) {
             self.impls_map.incoherent_impls.entry(simp).or_default().push(impl_def_id);
         } else {
             bug!("unexpected primitive type: {:?}", ty);
diff --git a/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs b/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs
index 7bca4edcc..ad76e2bed 100644
--- a/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs
@@ -302,7 +302,7 @@ impl<'tcx> InherentOverlapChecker<'tcx> {
                         .iter()
                         .flatten()
                         .map(|r| r.impl_blocks.len() as isize - avg as isize)
-                        .map(|v| v.abs() as usize)
+                        .map(|v| v.unsigned_abs())
                         .sum::<usize>();
                     s / connected_regions.len()
                 },
diff --git a/compiler/rustc_hir_analysis/src/coherence/mod.rs b/compiler/rustc_hir_analysis/src/coherence/mod.rs
index 23490bc09..465e787c9 100644
--- a/compiler/rustc_hir_analysis/src/coherence/mod.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/mod.rs
@@ -133,8 +133,8 @@ fn coherent_trait(tcx: TyCtxt<'_>, def_id: DefId) {
         check_impl(tcx, impl_def_id, trait_ref);
         check_object_overlap(tcx, impl_def_id, trait_ref);
 
-        tcx.sess.time("unsafety_checking", || unsafety::check_item(tcx, impl_def_id));
-        tcx.sess.time("orphan_checking", || tcx.ensure().orphan_check_impl(impl_def_id));
+        unsafety::check_item(tcx, impl_def_id);
+        tcx.ensure().orphan_check_impl(impl_def_id);
     }
 
     builtin::check_trait(tcx, def_id);
diff --git a/compiler/rustc_hir_analysis/src/coherence/orphan.rs b/compiler/rustc_hir_analysis/src/coherence/orphan.rs
index 1f2de3f21..47c47de8c 100644
--- a/compiler/rustc_hir_analysis/src/coherence/orphan.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/orphan.rs
@@ -478,10 +478,6 @@ fn lint_auto_trait_impl<'tcx>(
     trait_ref: ty::TraitRef<'tcx>,
     impl_def_id: LocalDefId,
 ) {
-    if tcx.impl_polarity(impl_def_id) != ImplPolarity::Positive {
-        return;
-    }
-
     assert_eq!(trait_ref.substs.len(), 1);
     let self_ty = trait_ref.self_ty();
     let (self_type_did, substs) = match self_ty.kind() {
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index 604d54caf..cbbaf8f85 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -20,7 +20,7 @@ use crate::errors;
 use hir::def::DefKind;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed, StashKey};
+use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, StashKey};
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::{self, Visitor};
@@ -64,6 +64,7 @@ pub fn provide(providers: &mut Providers) {
         predicates_defined_on,
         explicit_predicates_of: predicates_of::explicit_predicates_of,
         super_predicates_of: predicates_of::super_predicates_of,
+        implied_predicates_of: predicates_of::implied_predicates_of,
         super_predicates_that_define_assoc_type:
             predicates_of::super_predicates_that_define_assoc_type,
         trait_explicit_predicates_and_bounds: predicates_of::trait_explicit_predicates_and_bounds,
@@ -113,7 +114,7 @@ pub fn provide(providers: &mut Providers) {
 /// the AST (`hir::Generics`), recursively.
 pub struct ItemCtxt<'tcx> {
     tcx: TyCtxt<'tcx>,
-    item_def_id: DefId,
+    item_def_id: LocalDefId,
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -333,21 +334,11 @@ fn bad_placeholder<'tcx>(
     let kind = if kind.ends_with('s') { format!("{}es", kind) } else { format!("{}s", kind) };
 
     spans.sort();
-    let mut err = struct_span_err!(
-        tcx.sess,
-        spans.clone(),
-        E0121,
-        "the placeholder `_` is not allowed within types on item signatures for {}",
-        kind
-    );
-    for span in spans {
-        err.span_label(span, "not allowed in type signatures");
-    }
-    err
+    tcx.sess.create_err(errors::PlaceholderNotAllowedItemSignatures { spans, kind })
 }
 
 impl<'tcx> ItemCtxt<'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, item_def_id: DefId) -> ItemCtxt<'tcx> {
+    pub fn new(tcx: TyCtxt<'tcx>, item_def_id: LocalDefId) -> ItemCtxt<'tcx> {
         ItemCtxt { tcx, item_def_id }
     }
 
@@ -356,7 +347,7 @@ impl<'tcx> ItemCtxt<'tcx> {
     }
 
     pub fn hir_id(&self) -> hir::HirId {
-        self.tcx.hir().local_def_id_to_hir_id(self.item_def_id.expect_local())
+        self.tcx.hir().local_def_id_to_hir_id(self.item_def_id)
     }
 
     pub fn node(&self) -> hir::Node<'tcx> {
@@ -370,20 +361,16 @@ impl<'tcx> AstConv<'tcx> for ItemCtxt<'tcx> {
     }
 
     fn item_def_id(&self) -> DefId {
-        self.item_def_id
+        self.item_def_id.to_def_id()
     }
 
     fn get_type_parameter_bounds(
         &self,
         span: Span,
-        def_id: DefId,
+        def_id: LocalDefId,
         assoc_name: Ident,
     ) -> ty::GenericPredicates<'tcx> {
-        self.tcx.at(span).type_param_predicates((
-            self.item_def_id,
-            def_id.expect_local(),
-            assoc_name,
-        ))
+        self.tcx.at(span).type_param_predicates((self.item_def_id, def_id, assoc_name))
     }
 
     fn re_infer(&self, _: Option<&ty::GenericParamDef>, _: Span) -> Option<ty::Region<'tcx>> {
@@ -423,13 +410,8 @@ impl<'tcx> AstConv<'tcx> for ItemCtxt<'tcx> {
             self.tcx().mk_projection(item_def_id, item_substs)
         } else {
             // There are no late-bound regions; we can just ignore the binder.
-            let mut err = struct_span_err!(
-                self.tcx().sess,
-                span,
-                E0212,
-                "cannot use the associated type of a trait \
-                 with uninferred generic parameters"
-            );
+            let (mut mpart_sugg, mut inferred_sugg) = (None, None);
+            let mut bound = String::new();
 
             match self.node() {
                 hir::Node::Field(_) | hir::Node::Ctor(_) | hir::Node::Variant(_) => {
@@ -448,31 +430,25 @@ impl<'tcx> AstConv<'tcx> for ItemCtxt<'tcx> {
                                     (bound.span.shrink_to_lo(), format!("{}, ", lt_name))
                                 }
                             };
-                            let suggestions = vec![
-                                (lt_sp, sugg),
-                                (
-                                    span.with_hi(item_segment.ident.span.lo()),
-                                    format!(
-                                        "{}::",
-                                        // Replace the existing lifetimes with a new named lifetime.
-                                        self.tcx.replace_late_bound_regions_uncached(
-                                            poly_trait_ref,
-                                            |_| {
-                                                self.tcx.mk_re_early_bound(ty::EarlyBoundRegion {
-                                                    def_id: item_def_id,
-                                                    index: 0,
-                                                    name: Symbol::intern(&lt_name),
-                                                })
-                                            }
-                                        ),
+                            mpart_sugg = Some(errors::AssociatedTypeTraitUninferredGenericParamsMultipartSuggestion {
+                                fspan: lt_sp,
+                                first: sugg,
+                                sspan: span.with_hi(item_segment.ident.span.lo()),
+                                second: format!(
+                                    "{}::",
+                                    // Replace the existing lifetimes with a new named lifetime.
+                                    self.tcx.replace_late_bound_regions_uncached(
+                                        poly_trait_ref,
+                                        |_| {
+                                            self.tcx.mk_re_early_bound(ty::EarlyBoundRegion {
+                                                def_id: item_def_id,
+                                                index: 0,
+                                                name: Symbol::intern(&lt_name),
+                                            })
+                                        }
                                     ),
                                 ),
-                            ];
-                            err.multipart_suggestion(
-                                "use a fully qualified path with explicit lifetimes",
-                                suggestions,
-                                Applicability::MaybeIncorrect,
-                            );
+                            });
                         }
                         _ => {}
                     }
@@ -486,20 +462,23 @@ impl<'tcx> AstConv<'tcx> for ItemCtxt<'tcx> {
                 | hir::Node::ForeignItem(_)
                 | hir::Node::TraitItem(_)
                 | hir::Node::ImplItem(_) => {
-                    err.span_suggestion_verbose(
-                        span.with_hi(item_segment.ident.span.lo()),
-                        "use a fully qualified path with inferred lifetimes",
-                        format!(
-                            "{}::",
-                            // Erase named lt, we want `<A as B<'_>::C`, not `<A as B<'a>::C`.
-                            self.tcx.anonymize_bound_vars(poly_trait_ref).skip_binder(),
-                        ),
-                        Applicability::MaybeIncorrect,
+                    inferred_sugg = Some(span.with_hi(item_segment.ident.span.lo()));
+                    bound = format!(
+                        "{}::",
+                        // Erase named lt, we want `<A as B<'_>::C`, not `<A as B<'a>::C`.
+                        self.tcx.anonymize_bound_vars(poly_trait_ref).skip_binder(),
                     );
                 }
                 _ => {}
             }
-            self.tcx().ty_error(err.emit())
+            self.tcx().ty_error(self.tcx().sess.emit_err(
+                errors::AssociatedTypeTraitUninferredGenericParams {
+                    span,
+                    inferred_sugg,
+                    bound,
+                    mpart_sugg,
+                },
+            ))
         }
     }
 
@@ -618,6 +597,7 @@ fn convert_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
         }
         hir::ItemKind::TraitAlias(..) => {
             tcx.ensure().generics_of(def_id);
+            tcx.at(it.span).implied_predicates_of(def_id);
             tcx.at(it.span).super_predicates_of(def_id);
             tcx.ensure().predicates_of(def_id);
         }
@@ -767,14 +747,12 @@ fn convert_enum_variant_types(tcx: TyCtxt<'_>, def_id: DefId) {
                 Some(discr)
             } else {
                 let span = tcx.def_span(variant.def_id);
-                struct_span_err!(tcx.sess, span, E0370, "enum discriminant overflowed")
-                    .span_label(span, format!("overflowed on value after {}", prev_discr.unwrap()))
-                    .note(&format!(
-                        "explicitly set `{} = {}` if that is desired outcome",
-                        tcx.item_name(variant.def_id),
-                        wrapped_discr
-                    ))
-                    .emit();
+                tcx.sess.emit_err(errors::EnumDiscriminantOverflowed {
+                    span,
+                    discr: prev_discr.unwrap().to_string(),
+                    item_name: tcx.item_name(variant.def_id),
+                    wrapped_discr: wrapped_discr.to_string(),
+                });
                 None
             }
             .unwrap_or(wrapped_discr),
@@ -839,17 +817,15 @@ fn convert_variant(
         adt_kind,
         parent_did.to_def_id(),
         recovered,
-        adt_kind == AdtKind::Struct && tcx.has_attr(parent_did.to_def_id(), sym::non_exhaustive)
-            || variant_did.map_or(false, |variant_did| {
-                tcx.has_attr(variant_did.to_def_id(), sym::non_exhaustive)
-            }),
+        adt_kind == AdtKind::Struct && tcx.has_attr(parent_did, sym::non_exhaustive)
+            || variant_did
+                .map_or(false, |variant_did| tcx.has_attr(variant_did, sym::non_exhaustive)),
     )
 }
 
-fn adt_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AdtDef<'_> {
+fn adt_def(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::AdtDef<'_> {
     use rustc_hir::*;
 
-    let def_id = def_id.expect_local();
     let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
     let Node::Item(item) = tcx.hir().get(hir_id) else {
         bug!();
@@ -908,8 +884,8 @@ fn adt_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AdtDef<'_> {
     tcx.mk_adt_def(def_id.to_def_id(), kind, variants, repr)
 }
 
-fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
-    let item = tcx.hir().expect_item(def_id.expect_local());
+fn trait_def(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::TraitDef {
+    let item = tcx.hir().expect_item(def_id);
 
     let (is_auto, unsafety, items) = match item.kind {
         hir::ItemKind::Trait(is_auto, unsafety, .., items) => {
@@ -921,14 +897,7 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
 
     let paren_sugar = tcx.has_attr(def_id, sym::rustc_paren_sugar);
     if paren_sugar && !tcx.features().unboxed_closures {
-        tcx.sess
-            .struct_span_err(
-                item.span,
-                "the `#[rustc_paren_sugar]` attribute is a temporary means of controlling \
-                 which traits can use parenthetical notation",
-            )
-            .help("add `#![feature(unboxed_closures)]` to the crate attributes to use it")
-            .emit();
+        tcx.sess.emit_err(errors::ParenSugarAttribute { span: item.span });
     }
 
     let is_marker = tcx.has_attr(def_id, sym::marker);
@@ -948,13 +917,7 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
         // and that they are all identifiers
         .and_then(|attr| match attr.meta_item_list() {
             Some(items) if items.len() < 2 => {
-                tcx.sess
-                    .struct_span_err(
-                        attr.span,
-                        "the `#[rustc_must_implement_one_of]` attribute must be \
-                         used with at least 2 args",
-                    )
-                    .emit();
+                tcx.sess.emit_err(errors::MustImplementOneOfAttribute { span: attr.span });
 
                 None
             }
@@ -963,9 +926,7 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
                 .map(|item| item.ident().ok_or(item.span()))
                 .collect::<Result<Box<[_]>, _>>()
                 .map_err(|span| {
-                    tcx.sess
-                        .struct_span_err(span, "must be a name of an associated function")
-                        .emit();
+                    tcx.sess.emit_err(errors::MustBeNameOfAssociatedFunction { span });
                 })
                 .ok()
                 .zip(Some(attr.span)),
@@ -981,13 +942,10 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
                 match item {
                     Some(item) if matches!(item.kind, hir::AssocItemKind::Fn { .. }) => {
                         if !tcx.impl_defaultness(item.id.owner_id).has_value() {
-                            tcx.sess
-                                .struct_span_err(
-                                    item.span,
-                                    "function doesn't have a default implementation",
-                                )
-                                .span_note(attr_span, "required by this annotation")
-                                .emit();
+                            tcx.sess.emit_err(errors::FunctionNotHaveDefaultImplementation {
+                                span: item.span,
+                                note_span: attr_span,
+                            });
 
                             return Some(());
                         }
@@ -995,19 +953,14 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
                         return None;
                     }
                     Some(item) => {
-                        tcx.sess
-                            .struct_span_err(item.span, "not a function")
-                            .span_note(attr_span, "required by this annotation")
-                            .note(
-                                "all `#[rustc_must_implement_one_of]` arguments must be associated \
-                                 function names",
-                            )
-                            .emit();
+                        tcx.sess.emit_err(errors::MustImplementNotFunction {
+                            span: item.span,
+                            span_note: errors::MustImplementNotFunctionSpanNote { span: attr_span },
+                            note: errors::MustImplementNotFunctionNote {},
+                        });
                     }
                     None => {
-                        tcx.sess
-                            .struct_span_err(ident.span, "function not found in this trait")
-                            .emit();
+                        tcx.sess.emit_err(errors::FunctionNotFoundInTrait { span: ident.span });
                     }
                 }
 
@@ -1024,9 +977,7 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
             for ident in &*list {
                 if let Some(dup) = set.insert(ident.name, ident.span) {
                     tcx.sess
-                        .struct_span_err(vec![dup, ident.span], "functions names are duplicated")
-                        .note("all `#[rustc_must_implement_one_of]` arguments must be unique")
-                        .emit();
+                        .emit_err(errors::FunctionNamesDuplicated { spans: vec![dup, ident.span] });
 
                     no_dups = false;
                 }
@@ -1036,7 +987,7 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
         });
 
     ty::TraitDef {
-        def_id,
+        def_id: def_id.to_def_id(),
         unsafety,
         paren_sugar,
         has_auto_impl: is_auto,
@@ -1091,14 +1042,13 @@ pub fn get_infer_ret_ty<'hir>(output: &'hir hir::FnRetTy<'hir>) -> Option<&'hir
 }
 
 #[instrument(level = "debug", skip(tcx))]
-fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<ty::PolyFnSig<'_>> {
+fn fn_sig(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<ty::PolyFnSig<'_>> {
     use rustc_hir::Node::*;
     use rustc_hir::*;
 
-    let def_id = def_id.expect_local();
     let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
 
-    let icx = ItemCtxt::new(tcx, def_id.to_def_id());
+    let icx = ItemCtxt::new(tcx, def_id);
 
     let output = match tcx.hir().get(hir_id) {
         TraitItem(hir::TraitItem {
@@ -1139,7 +1089,7 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<ty::PolyFnSig<'_>>
 
         ForeignItem(&hir::ForeignItem { kind: ForeignItemKind::Fn(fn_decl, _, _), .. }) => {
             let abi = tcx.hir().get_foreign_abi(hir_id);
-            compute_sig_of_foreign_fn_decl(tcx, def_id.to_def_id(), fn_decl, abi)
+            compute_sig_of_foreign_fn_decl(tcx, def_id, fn_decl, abi)
         }
 
         Ctor(data) | Variant(hir::Variant { data, .. }) if data.ctor().is_some() => {
@@ -1215,7 +1165,7 @@ fn infer_return_ty_for_fn_sig<'tcx>(
                     fn_sig,
                     Applicability::MachineApplicable,
                 );
-            } else if let Some(sugg) = suggest_impl_trait(tcx, ret_ty, ty.span, hir_id, def_id) {
+            } else if let Some(sugg) = suggest_impl_trait(tcx, ret_ty, ty.span, def_id) {
                 diag.span_suggestion(
                     ty.span,
                     "replace with an appropriate return type",
@@ -1247,12 +1197,10 @@ fn infer_return_ty_for_fn_sig<'tcx>(
     }
 }
 
-// FIXME(vincenzopalazzo): remove the hir item when the refactoring is stable
 fn suggest_impl_trait<'tcx>(
     tcx: TyCtxt<'tcx>,
     ret_ty: Ty<'tcx>,
     span: Span,
-    _hir_id: hir::HirId,
     def_id: LocalDefId,
 ) -> Option<String> {
     let format_as_assoc: fn(_, _, _, _, _) -> _ =
@@ -1338,9 +1286,12 @@ fn suggest_impl_trait<'tcx>(
     None
 }
 
-fn impl_trait_ref(tcx: TyCtxt<'_>, def_id: DefId) -> Option<ty::EarlyBinder<ty::TraitRef<'_>>> {
+fn impl_trait_ref(
+    tcx: TyCtxt<'_>,
+    def_id: LocalDefId,
+) -> Option<ty::EarlyBinder<ty::TraitRef<'_>>> {
     let icx = ItemCtxt::new(tcx, def_id);
-    let impl_ = tcx.hir().expect_item(def_id.expect_local()).expect_impl();
+    let impl_ = tcx.hir().expect_item(def_id).expect_impl();
     impl_
         .of_trait
         .as_ref()
@@ -1380,9 +1331,9 @@ fn check_impl_constness(
     }
 }
 
-fn impl_polarity(tcx: TyCtxt<'_>, def_id: DefId) -> ty::ImplPolarity {
+fn impl_polarity(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::ImplPolarity {
     let is_rustc_reservation = tcx.has_attr(def_id, sym::rustc_reservation_impl);
-    let item = tcx.hir().expect_item(def_id.expect_local());
+    let item = tcx.hir().expect_item(def_id);
     match &item.kind {
         hir::ItemKind::Impl(hir::Impl {
             polarity: hir::ImplPolarity::Negative(span),
@@ -1465,16 +1416,16 @@ fn predicates_defined_on(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericPredicate
 
 fn compute_sig_of_foreign_fn_decl<'tcx>(
     tcx: TyCtxt<'tcx>,
-    def_id: DefId,
+    def_id: LocalDefId,
     decl: &'tcx hir::FnDecl<'tcx>,
     abi: abi::Abi,
 ) -> ty::PolyFnSig<'tcx> {
     let unsafety = if abi == abi::Abi::RustIntrinsic {
-        intrinsic_operation_unsafety(tcx, def_id)
+        intrinsic_operation_unsafety(tcx, def_id.to_def_id())
     } else {
         hir::Unsafety::Unsafe
     };
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
     let fty =
         ItemCtxt::new(tcx, def_id).astconv().ty_of_fn(hir_id, unsafety, abi, decl, None, None);
 
@@ -1491,17 +1442,7 @@ fn compute_sig_of_foreign_fn_decl<'tcx>(
                     .source_map()
                     .span_to_snippet(ast_ty.span)
                     .map_or_else(|_| String::new(), |s| format!(" `{}`", s));
-                tcx.sess
-                    .struct_span_err(
-                        ast_ty.span,
-                        &format!(
-                            "use of SIMD type{} in FFI is highly experimental and \
-                             may result in invalid code",
-                            snip
-                        ),
-                    )
-                    .help("add `#![feature(simd_ffi)]` to the crate attributes to enable")
-                    .emit();
+                tcx.sess.emit_err(errors::SIMDFFIHighlyExperimental { span: ast_ty.span, snip });
             }
         };
         for (input, ty) in iter::zip(decl.inputs, fty.inputs().skip_binder()) {
@@ -1515,31 +1456,28 @@ fn compute_sig_of_foreign_fn_decl<'tcx>(
     fty
 }
 
-fn is_foreign_item(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
-    match tcx.hir().get_if_local(def_id) {
-        Some(Node::ForeignItem(..)) => true,
-        Some(_) => false,
-        _ => bug!("is_foreign_item applied to non-local def-id {:?}", def_id),
+fn is_foreign_item(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
+    match tcx.hir().get_by_def_id(def_id) {
+        Node::ForeignItem(..) => true,
+        _ => false,
     }
 }
 
-fn generator_kind(tcx: TyCtxt<'_>, def_id: DefId) -> Option<hir::GeneratorKind> {
-    match tcx.hir().get_if_local(def_id) {
-        Some(Node::Expr(&rustc_hir::Expr {
+fn generator_kind(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<hir::GeneratorKind> {
+    match tcx.hir().get_by_def_id(def_id) {
+        Node::Expr(&rustc_hir::Expr {
             kind: rustc_hir::ExprKind::Closure(&rustc_hir::Closure { body, .. }),
             ..
-        })) => tcx.hir().body(body).generator_kind(),
-        Some(_) => None,
-        _ => bug!("generator_kind applied to non-local def-id {:?}", def_id),
+        }) => tcx.hir().body(body).generator_kind(),
+        _ => None,
     }
 }
 
-fn is_type_alias_impl_trait<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
-    match tcx.hir().get_if_local(def_id) {
-        Some(Node::Item(hir::Item { kind: hir::ItemKind::OpaqueTy(opaque), .. })) => {
+fn is_type_alias_impl_trait<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> bool {
+    match tcx.hir().get_by_def_id(def_id) {
+        Node::Item(hir::Item { kind: hir::ItemKind::OpaqueTy(opaque), .. }) => {
             matches!(opaque.origin, hir::OpaqueTyOrigin::TyAlias)
         }
-        Some(_) => bug!("tried getting opaque_ty_origin for non-opaque: {:?}", def_id),
-        _ => bug!("tried getting opaque_ty_origin for non-local def-id {:?}", def_id),
+        _ => bug!("tried getting opaque_ty_origin for non-opaque: {:?}", def_id),
     }
 }
diff --git a/compiler/rustc_hir_analysis/src/collect/generics_of.rs b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
index 127d4fa90..119933697 100644
--- a/compiler/rustc_hir_analysis/src/collect/generics_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
@@ -5,16 +5,16 @@ use hir::{
 };
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::LocalDefId;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::lint;
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::Span;
 
-pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics {
+pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
     use rustc_hir::*;
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
 
     let node = tcx.hir().get(hir_id);
     let parent_def_id = match node {
@@ -121,7 +121,7 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics {
                         Some(parent_def_id.to_def_id())
                     }
                     Node::Expr(&Expr { kind: ExprKind::ConstBlock(_), .. }) => {
-                        Some(tcx.typeck_root_def_id(def_id))
+                        Some(tcx.typeck_root_def_id(def_id.to_def_id()))
                     }
                     // Exclude `GlobalAsm` here which cannot have generics.
                     Node::Expr(&Expr { kind: ExprKind::InlineAsm(asm), .. })
@@ -140,7 +140,7 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics {
             }
         }
         Node::Expr(&hir::Expr { kind: hir::ExprKind::Closure { .. }, .. }) => {
-            Some(tcx.typeck_root_def_id(def_id))
+            Some(tcx.typeck_root_def_id(def_id.to_def_id()))
         }
         Node::Item(item) => match item.kind {
             ItemKind::OpaqueTy(hir::OpaqueTy {
@@ -189,7 +189,7 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics {
                     let opt_self = Some(ty::GenericParamDef {
                         index: 0,
                         name: kw::SelfUpper,
-                        def_id,
+                        def_id: def_id.to_def_id(),
                         pure_wrt_drop: false,
                         kind: ty::GenericParamDefKind::Type {
                             has_default: false,
@@ -326,7 +326,7 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics {
         params.extend(dummy_args.iter().map(|&arg| ty::GenericParamDef {
             index: next_index(),
             name: Symbol::intern(arg),
-            def_id,
+            def_id: def_id.to_def_id(),
             pure_wrt_drop: false,
             kind: ty::GenericParamDefKind::Type { has_default: false, synthetic: false },
         }));
@@ -339,7 +339,7 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics {
             params.push(ty::GenericParamDef {
                 index: next_index(),
                 name: Symbol::intern("<const_ty>"),
-                def_id,
+                def_id: def_id.to_def_id(),
                 pure_wrt_drop: false,
                 kind: ty::GenericParamDefKind::Type { has_default: false, synthetic: false },
             });
diff --git a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
index 9cf3ff65a..2e56d2463 100644
--- a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
+++ b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
@@ -3,8 +3,8 @@ use crate::astconv::AstConv;
 use rustc_hir as hir;
 use rustc_infer::traits::util;
 use rustc_middle::ty::subst::InternalSubsts;
-use rustc_middle::ty::{self, DefIdTree, TyCtxt};
-use rustc_span::def_id::DefId;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::Span;
 
 /// For associated types we include both bounds written on the type
@@ -16,12 +16,12 @@ use rustc_span::Span;
 /// `hr-associated-type-bound-1.rs`.
 fn associated_type_bounds<'tcx>(
     tcx: TyCtxt<'tcx>,
-    assoc_item_def_id: DefId,
+    assoc_item_def_id: LocalDefId,
     ast_bounds: &'tcx [hir::GenericBound<'tcx>],
     span: Span,
 ) -> &'tcx [(ty::Predicate<'tcx>, Span)] {
     let item_ty = tcx.mk_projection(
-        assoc_item_def_id,
+        assoc_item_def_id.to_def_id(),
         InternalSubsts::identity_for_item(tcx, assoc_item_def_id),
     );
 
@@ -30,8 +30,8 @@ fn associated_type_bounds<'tcx>(
     // Associated types are implicitly sized unless a `?Sized` bound is found
     icx.astconv().add_implicitly_sized(&mut bounds, item_ty, ast_bounds, None, span);
 
-    let trait_def_id = tcx.parent(assoc_item_def_id);
-    let trait_predicates = tcx.trait_explicit_predicates_and_bounds(trait_def_id.expect_local());
+    let trait_def_id = tcx.local_parent(assoc_item_def_id);
+    let trait_predicates = tcx.trait_explicit_predicates_and_bounds(trait_def_id);
 
     let bounds_from_parent = trait_predicates.predicates.iter().copied().filter(|(pred, _)| {
         match pred.kind().skip_binder() {
@@ -45,7 +45,11 @@ fn associated_type_bounds<'tcx>(
     });
 
     let all_bounds = tcx.arena.alloc_from_iter(bounds.predicates().chain(bounds_from_parent));
-    debug!("associated_type_bounds({}) = {:?}", tcx.def_path_str(assoc_item_def_id), all_bounds);
+    debug!(
+        "associated_type_bounds({}) = {:?}",
+        tcx.def_path_str(assoc_item_def_id.to_def_id()),
+        all_bounds
+    );
     all_bounds
 }
 
@@ -56,19 +60,12 @@ fn associated_type_bounds<'tcx>(
 #[instrument(level = "trace", skip(tcx), ret)]
 fn opaque_type_bounds<'tcx>(
     tcx: TyCtxt<'tcx>,
-    opaque_def_id: DefId,
+    opaque_def_id: LocalDefId,
     ast_bounds: &'tcx [hir::GenericBound<'tcx>],
+    item_ty: Ty<'tcx>,
     span: Span,
-    in_trait: bool,
 ) -> &'tcx [(ty::Predicate<'tcx>, Span)] {
     ty::print::with_no_queries!({
-        let substs = InternalSubsts::identity_for_item(tcx, opaque_def_id);
-        let item_ty = if in_trait {
-            tcx.mk_projection(opaque_def_id, substs)
-        } else {
-            tcx.mk_opaque(opaque_def_id, substs)
-        };
-
         let icx = ItemCtxt::new(tcx, opaque_def_id);
         let mut bounds = icx.astconv().compute_bounds(item_ty, ast_bounds);
         // Opaque types are implicitly sized unless a `?Sized` bound is found
@@ -81,9 +78,31 @@ fn opaque_type_bounds<'tcx>(
 
 pub(super) fn explicit_item_bounds(
     tcx: TyCtxt<'_>,
-    def_id: DefId,
+    def_id: LocalDefId,
 ) -> &'_ [(ty::Predicate<'_>, Span)] {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+    match tcx.opt_rpitit_info(def_id.to_def_id()) {
+        // RPITIT's bounds are the same as opaque type bounds, but with
+        // a projection self type.
+        Some(ty::ImplTraitInTraitData::Trait { opaque_def_id, .. }) => {
+            let item = tcx.hir().get_by_def_id(opaque_def_id.expect_local()).expect_item();
+            let opaque_ty = item.expect_opaque_ty();
+            return opaque_type_bounds(
+                tcx,
+                opaque_def_id.expect_local(),
+                opaque_ty.bounds,
+                tcx.mk_projection(
+                    def_id.to_def_id(),
+                    ty::InternalSubsts::identity_for_item(tcx, def_id),
+                ),
+                item.span,
+            );
+        }
+        // These should have been fed!
+        Some(ty::ImplTraitInTraitData::Impl { .. }) => unreachable!(),
+        None => {}
+    }
+
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
     match tcx.hir().get(hir_id) {
         hir::Node::TraitItem(hir::TraitItem {
             kind: hir::TraitItemKind::Type(bounds, _),
@@ -94,7 +113,15 @@ pub(super) fn explicit_item_bounds(
             kind: hir::ItemKind::OpaqueTy(hir::OpaqueTy { bounds, in_trait, .. }),
             span,
             ..
-        }) => opaque_type_bounds(tcx, def_id, bounds, *span, *in_trait),
+        }) => {
+            let substs = InternalSubsts::identity_for_item(tcx, def_id);
+            let item_ty = if *in_trait && !tcx.lower_impl_trait_in_trait_to_assoc_ty() {
+                tcx.mk_projection(def_id.to_def_id(), substs)
+            } else {
+                tcx.mk_opaque(def_id.to_def_id(), substs)
+            };
+            opaque_type_bounds(tcx, def_id, bounds, item_ty, *span)
+        }
         _ => bug!("item_bounds called on {:?}", def_id),
     }
 }
@@ -103,12 +130,9 @@ pub(super) fn item_bounds(
     tcx: TyCtxt<'_>,
     def_id: DefId,
 ) -> ty::EarlyBinder<&'_ ty::List<ty::Predicate<'_>>> {
-    let bounds = tcx.mk_predicates_from_iter(
-        util::elaborate_predicates(
-            tcx,
-            tcx.explicit_item_bounds(def_id).iter().map(|&(bound, _span)| bound),
-        )
-        .map(|obligation| obligation.predicate),
-    );
+    let bounds = tcx.mk_predicates_from_iter(util::elaborate(
+        tcx,
+        tcx.explicit_item_bounds(def_id).iter().map(|&(bound, _span)| bound),
+    ));
     ty::EarlyBinder(bounds)
 }
diff --git a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
index 2badd66e3..9358ed612 100644
--- a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
@@ -62,15 +62,16 @@ pub(super) fn predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericPredic
 /// Returns a list of user-specified type predicates for the definition with ID `def_id`.
 /// N.B., this does not include any implied/inferred constraints.
 #[instrument(level = "trace", skip(tcx), ret)]
-fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericPredicates<'_> {
+fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::GenericPredicates<'_> {
     use rustc_hir::*;
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
     let node = tcx.hir().get(hir_id);
 
     let mut is_trait = None;
     let mut is_default_impl_trait = None;
 
+    // FIXME: Should ItemCtxt take a LocalDefId?
     let icx = ItemCtxt::new(tcx, def_id);
 
     const NO_GENERICS: &hir::Generics<'_> = hir::Generics::empty();
@@ -99,7 +100,7 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
             | ItemKind::Union(_, generics) => generics,
 
             ItemKind::Trait(_, _, generics, ..) | ItemKind::TraitAlias(generics, _) => {
-                is_trait = Some(ty::TraitRef::identity(tcx, def_id));
+                is_trait = Some(ty::TraitRef::identity(tcx, def_id.to_def_id()));
                 generics
             }
             ItemKind::OpaqueTy(OpaqueTy { generics, .. }) => generics,
@@ -124,7 +125,7 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
     // on a trait we need to add in the supertrait bounds and bounds found on
     // associated types.
     if let Some(_trait_ref) = is_trait {
-        predicates.extend(tcx.super_predicates_of(def_id).predicates.iter().cloned());
+        predicates.extend(tcx.implied_predicates_of(def_id).predicates.iter().cloned());
     }
 
     // In default impls, we can assume that the self type implements
@@ -253,7 +254,7 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
     }
 
     if tcx.features().generic_const_exprs {
-        predicates.extend(const_evaluatable_predicates_of(tcx, def_id.expect_local()));
+        predicates.extend(const_evaluatable_predicates_of(tcx, def_id));
     }
 
     let mut predicates: Vec<_> = predicates.into_iter().collect();
@@ -392,18 +393,18 @@ pub(super) fn trait_explicit_predicates_and_bounds(
     def_id: LocalDefId,
 ) -> ty::GenericPredicates<'_> {
     assert_eq!(tcx.def_kind(def_id), DefKind::Trait);
-    gather_explicit_predicates_of(tcx, def_id.to_def_id())
+    gather_explicit_predicates_of(tcx, def_id)
 }
 
 pub(super) fn explicit_predicates_of<'tcx>(
     tcx: TyCtxt<'tcx>,
-    def_id: DefId,
+    def_id: LocalDefId,
 ) -> ty::GenericPredicates<'tcx> {
     let def_kind = tcx.def_kind(def_id);
     if let DefKind::Trait = def_kind {
         // Remove bounds on associated types from the predicates, they will be
         // returned by `explicit_item_bounds`.
-        let predicates_and_bounds = tcx.trait_explicit_predicates_and_bounds(def_id.expect_local());
+        let predicates_and_bounds = tcx.trait_explicit_predicates_and_bounds(def_id);
         let trait_identity_substs = InternalSubsts::identity_for_item(tcx, def_id);
 
         let is_assoc_item_ty = |ty: Ty<'tcx>| {
@@ -418,7 +419,8 @@ pub(super) fn explicit_predicates_of<'tcx>(
             //   supertrait).
             if let ty::Alias(ty::Projection, projection) = ty.kind() {
                 projection.substs == trait_identity_substs
-                    && tcx.associated_item(projection.def_id).container_id(tcx) == def_id
+                    && tcx.associated_item(projection.def_id).container_id(tcx)
+                        == def_id.to_def_id()
             } else {
                 false
             }
@@ -449,7 +451,7 @@ pub(super) fn explicit_predicates_of<'tcx>(
         }
     } else {
         if matches!(def_kind, DefKind::AnonConst) && tcx.lazy_normalization() {
-            let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+            let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
             let parent_def_id = tcx.hir().get_parent_item(hir_id);
 
             if let Some(defaulted_param_def_id) =
@@ -532,89 +534,136 @@ pub(super) fn explicit_predicates_of<'tcx>(
     }
 }
 
+#[derive(Copy, Clone, Debug)]
+pub enum PredicateFilter {
+    /// All predicates may be implied by the trait
+    All,
+
+    /// Only traits that reference `Self: ..` are implied by the trait
+    SelfOnly,
+
+    /// Only traits that reference `Self: ..` and define an associated type
+    /// with the given ident are implied by the trait
+    SelfThatDefines(Ident),
+}
+
 /// Ensures that the super-predicates of the trait with a `DefId`
 /// of `trait_def_id` are converted and stored. This also ensures that
 /// the transitive super-predicates are converted.
 pub(super) fn super_predicates_of(
     tcx: TyCtxt<'_>,
-    trait_def_id: DefId,
+    trait_def_id: LocalDefId,
+) -> ty::GenericPredicates<'_> {
+    implied_predicates_with_filter(tcx, trait_def_id.to_def_id(), PredicateFilter::SelfOnly)
+}
+
+pub(super) fn super_predicates_that_define_assoc_type(
+    tcx: TyCtxt<'_>,
+    (trait_def_id, assoc_name): (DefId, Ident),
+) -> ty::GenericPredicates<'_> {
+    implied_predicates_with_filter(tcx, trait_def_id, PredicateFilter::SelfThatDefines(assoc_name))
+}
+
+pub(super) fn implied_predicates_of(
+    tcx: TyCtxt<'_>,
+    trait_def_id: LocalDefId,
 ) -> ty::GenericPredicates<'_> {
-    tcx.super_predicates_that_define_assoc_type((trait_def_id, None))
+    if tcx.is_trait_alias(trait_def_id.to_def_id()) {
+        implied_predicates_with_filter(tcx, trait_def_id.to_def_id(), PredicateFilter::All)
+    } else {
+        tcx.super_predicates_of(trait_def_id)
+    }
 }
 
 /// Ensures that the super-predicates of the trait with a `DefId`
 /// of `trait_def_id` are converted and stored. This also ensures that
 /// the transitive super-predicates are converted.
-pub(super) fn super_predicates_that_define_assoc_type(
+pub(super) fn implied_predicates_with_filter(
     tcx: TyCtxt<'_>,
-    (trait_def_id, assoc_name): (DefId, Option<Ident>),
+    trait_def_id: DefId,
+    filter: PredicateFilter,
 ) -> ty::GenericPredicates<'_> {
-    if trait_def_id.is_local() {
-        debug!("local trait");
-        let trait_hir_id = tcx.hir().local_def_id_to_hir_id(trait_def_id.expect_local());
-
-        let Node::Item(item) = tcx.hir().get(trait_hir_id) else {
-            bug!("trait_node_id {} is not an item", trait_hir_id);
-        };
+    let Some(trait_def_id) = trait_def_id.as_local() else {
+        // if `assoc_name` is None, then the query should've been redirected to an
+        // external provider
+        assert!(matches!(filter, PredicateFilter::SelfThatDefines(_)));
+        return tcx.super_predicates_of(trait_def_id);
+    };
 
-        let (generics, bounds) = match item.kind {
-            hir::ItemKind::Trait(.., generics, supertraits, _) => (generics, supertraits),
-            hir::ItemKind::TraitAlias(generics, supertraits) => (generics, supertraits),
-            _ => span_bug!(item.span, "super_predicates invoked on non-trait"),
-        };
+    let trait_hir_id = tcx.hir().local_def_id_to_hir_id(trait_def_id);
 
-        let icx = ItemCtxt::new(tcx, trait_def_id);
+    let Node::Item(item) = tcx.hir().get(trait_hir_id) else {
+        bug!("trait_node_id {} is not an item", trait_hir_id);
+    };
 
-        // Convert the bounds that follow the colon, e.g., `Bar + Zed` in `trait Foo: Bar + Zed`.
-        let self_param_ty = tcx.types.self_param;
-        let superbounds1 = if let Some(assoc_name) = assoc_name {
-            icx.astconv().compute_bounds_that_match_assoc_type(self_param_ty, bounds, assoc_name)
-        } else {
-            icx.astconv().compute_bounds(self_param_ty, bounds)
-        };
+    let (generics, bounds) = match item.kind {
+        hir::ItemKind::Trait(.., generics, supertraits, _) => (generics, supertraits),
+        hir::ItemKind::TraitAlias(generics, supertraits) => (generics, supertraits),
+        _ => span_bug!(item.span, "super_predicates invoked on non-trait"),
+    };
 
-        let superbounds1 = superbounds1.predicates();
-
-        // Convert any explicit superbounds in the where-clause,
-        // e.g., `trait Foo where Self: Bar`.
-        // In the case of trait aliases, however, we include all bounds in the where-clause,
-        // so e.g., `trait Foo = where u32: PartialEq<Self>` would include `u32: PartialEq<Self>`
-        // as one of its "superpredicates".
-        let is_trait_alias = tcx.is_trait_alias(trait_def_id);
-        let superbounds2 = icx.type_parameter_bounds_in_generics(
-            generics,
-            item.owner_id.def_id,
-            self_param_ty,
-            OnlySelfBounds(!is_trait_alias),
-            assoc_name,
-        );
+    let icx = ItemCtxt::new(tcx, trait_def_id);
+
+    let self_param_ty = tcx.types.self_param;
+    let (superbounds, where_bounds_that_match) = match filter {
+        PredicateFilter::All => (
+            // Convert the bounds that follow the colon (or equal in trait aliases)
+            icx.astconv().compute_bounds(self_param_ty, bounds),
+            // Also include all where clause bounds
+            icx.type_parameter_bounds_in_generics(
+                generics,
+                item.owner_id.def_id,
+                self_param_ty,
+                OnlySelfBounds(false),
+                None,
+            ),
+        ),
+        PredicateFilter::SelfOnly => (
+            // Convert the bounds that follow the colon (or equal in trait aliases)
+            icx.astconv().compute_bounds(self_param_ty, bounds),
+            // Include where clause bounds for `Self`
+            icx.type_parameter_bounds_in_generics(
+                generics,
+                item.owner_id.def_id,
+                self_param_ty,
+                OnlySelfBounds(true),
+                None,
+            ),
+        ),
+        PredicateFilter::SelfThatDefines(assoc_name) => (
+            // Convert the bounds that follow the colon (or equal) that reference the associated name
+            icx.astconv().compute_bounds_that_match_assoc_type(self_param_ty, bounds, assoc_name),
+            // Include where clause bounds for `Self` that reference the associated name
+            icx.type_parameter_bounds_in_generics(
+                generics,
+                item.owner_id.def_id,
+                self_param_ty,
+                OnlySelfBounds(true),
+                Some(assoc_name),
+            ),
+        ),
+    };
 
-        // Combine the two lists to form the complete set of superbounds:
-        let superbounds = &*tcx.arena.alloc_from_iter(superbounds1.into_iter().chain(superbounds2));
-        debug!(?superbounds);
+    // Combine the two lists to form the complete set of superbounds:
+    let implied_bounds = &*tcx
+        .arena
+        .alloc_from_iter(superbounds.predicates().into_iter().chain(where_bounds_that_match));
+    debug!(?implied_bounds);
 
+    // Now require that immediate supertraits are converted,
+    // which will, in turn, reach indirect supertraits.
+    if matches!(filter, PredicateFilter::SelfOnly) {
         // Now require that immediate supertraits are converted,
         // which will, in turn, reach indirect supertraits.
-        if assoc_name.is_none() {
-            // Now require that immediate supertraits are converted,
-            // which will, in turn, reach indirect supertraits.
-            for &(pred, span) in superbounds {
-                debug!("superbound: {:?}", pred);
-                if let ty::PredicateKind::Clause(ty::Clause::Trait(bound)) =
-                    pred.kind().skip_binder()
-                {
-                    tcx.at(span).super_predicates_of(bound.def_id());
-                }
+        for &(pred, span) in implied_bounds {
+            debug!("superbound: {:?}", pred);
+            if let ty::PredicateKind::Clause(ty::Clause::Trait(bound)) = pred.kind().skip_binder() {
+                tcx.at(span).super_predicates_of(bound.def_id());
             }
         }
-
-        ty::GenericPredicates { parent: None, predicates: superbounds }
-    } else {
-        // if `assoc_name` is None, then the query should've been redirected to an
-        // external provider
-        assert!(assoc_name.is_some());
-        tcx.super_predicates_of(trait_def_id)
     }
+
+    ty::GenericPredicates { parent: None, predicates: implied_bounds }
 }
 
 /// Returns the predicates defined on `item_def_id` of the form
@@ -622,7 +671,7 @@ pub(super) fn super_predicates_that_define_assoc_type(
 #[instrument(level = "trace", skip(tcx))]
 pub(super) fn type_param_predicates(
     tcx: TyCtxt<'_>,
-    (item_def_id, def_id, assoc_name): (DefId, LocalDefId, Ident),
+    (item_def_id, def_id, assoc_name): (LocalDefId, LocalDefId, Ident),
 ) -> ty::GenericPredicates<'_> {
     use rustc_hir::*;
 
@@ -637,21 +686,21 @@ pub(super) fn type_param_predicates(
     let ty = tcx.mk_ty_param(index, tcx.hir().ty_param_name(def_id));
 
     // Don't look for bounds where the type parameter isn't in scope.
-    let parent = if item_def_id == param_owner.to_def_id() {
+    let parent = if item_def_id == param_owner {
         None
     } else {
-        tcx.generics_of(item_def_id).parent
+        tcx.generics_of(item_def_id).parent.map(|def_id| def_id.expect_local())
     };
 
     let mut result = parent
         .map(|parent| {
             let icx = ItemCtxt::new(tcx, parent);
-            icx.get_type_parameter_bounds(DUMMY_SP, def_id.to_def_id(), assoc_name)
+            icx.get_type_parameter_bounds(DUMMY_SP, def_id, assoc_name)
         })
         .unwrap_or_default();
     let mut extend = None;
 
-    let item_hir_id = tcx.hir().local_def_id_to_hir_id(item_def_id.expect_local());
+    let item_hir_id = tcx.hir().local_def_id_to_hir_id(item_def_id);
     let ast_generics = match tcx.hir().get(item_hir_id) {
         Node::TraitItem(item) => &item.generics,
 
@@ -673,7 +722,8 @@ pub(super) fn type_param_predicates(
                 ItemKind::Trait(_, _, generics, ..) => {
                     // Implied `Self: Trait` and supertrait bounds.
                     if param_id == item_hir_id {
-                        let identity_trait_ref = ty::TraitRef::identity(tcx, item_def_id);
+                        let identity_trait_ref =
+                            ty::TraitRef::identity(tcx, item_def_id.to_def_id());
                         extend =
                             Some((identity_trait_ref.without_const().to_predicate(tcx), item.span));
                     }
diff --git a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
index 65a9052a6..e758fe95d 100644
--- a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
+++ b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
@@ -17,7 +17,7 @@ use rustc_hir::{GenericArg, GenericParam, GenericParamKind, HirIdMap, LifetimeNa
 use rustc_middle::bug;
 use rustc_middle::hir::nested_filter;
 use rustc_middle::middle::resolve_bound_vars::*;
-use rustc_middle::ty::{self, DefIdTree, TyCtxt, TypeSuperVisitable, TypeVisitor};
+use rustc_middle::ty::{self, TyCtxt, TypeSuperVisitable, TypeVisitor};
 use rustc_session::lint;
 use rustc_span::def_id::DefId;
 use rustc_span::symbol::{sym, Ident};
@@ -1051,9 +1051,8 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
     }
 }
 
-fn object_lifetime_default(tcx: TyCtxt<'_>, param_def_id: DefId) -> ObjectLifetimeDefault {
+fn object_lifetime_default(tcx: TyCtxt<'_>, param_def_id: LocalDefId) -> ObjectLifetimeDefault {
     debug_assert_eq!(tcx.def_kind(param_def_id), DefKind::TyParam);
-    let param_def_id = param_def_id.expect_local();
     let hir::Node::GenericParam(param) = tcx.hir().get_by_def_id(param_def_id) else {
         bug!("expected GenericParam for object_lifetime_default");
     };
@@ -1427,25 +1426,25 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
             if let ResolvedArg::LateBound(..) = def && crossed_anon_const {
                 let use_span = self.tcx.hir().span(hir_id);
                 let def_span = self.tcx.def_span(param_def_id);
-                match self.tcx.def_kind(param_def_id) {
+                let guar = match self.tcx.def_kind(param_def_id) {
                     DefKind::ConstParam => {
                         self.tcx.sess.emit_err(errors::CannotCaptureLateBoundInAnonConst::Const {
                             use_span,
                             def_span,
-                        });
+                        })
                     }
                     DefKind::TyParam => {
                         self.tcx.sess.emit_err(errors::CannotCaptureLateBoundInAnonConst::Type {
                             use_span,
                             def_span,
-                        });
+                        })
                     }
                     _ => unreachable!(),
-                }
-                return;
+                };
+                self.map.defs.insert(hir_id, ResolvedArg::Error(guar));
+            } else {
+                self.map.defs.insert(hir_id, def);
             }
-
-            self.map.defs.insert(hir_id, def);
             return;
         }
 
@@ -1462,7 +1461,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
         depth: usize,
         generic_args: &'tcx hir::GenericArgs<'tcx>,
     ) {
-        if generic_args.parenthesized {
+        if generic_args.parenthesized == hir::GenericArgsParentheses::ParenSugar {
             self.visit_fn_like_elision(
                 generic_args.inputs(),
                 Some(generic_args.bindings[0].ty()),
@@ -1641,7 +1640,59 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
                 },
                 s: self.scope,
             };
-            if let Some(type_def_id) = type_def_id {
+            // If the binding is parenthesized, then this must be `feature(return_type_notation)`.
+            // In that case, introduce a binder over all of the function's early and late bound vars.
+            //
+            // For example, given
+            // ```
+            // trait Foo {
+            //     async fn x<'r, T>();
+            // }
+            // ```
+            // and a bound that looks like:
+            //    `for<'a> T::Trait<'a, x(): for<'b> Other<'b>>`
+            // this is going to expand to something like:
+            //    `for<'a> for<'r, T> <T as Trait<'a>>::x::<'r, T>::{opaque#0}: for<'b> Other<'b>`.
+            if binding.gen_args.parenthesized == hir::GenericArgsParentheses::ReturnTypeNotation {
+                let bound_vars = if let Some(type_def_id) = type_def_id
+                    && self.tcx.def_kind(type_def_id) == DefKind::Trait
+                    // FIXME(return_type_notation): We could bound supertrait methods.
+                    && let Some(assoc_fn) = self
+                        .tcx
+                        .associated_items(type_def_id)
+                        .find_by_name_and_kind(self.tcx, binding.ident, ty::AssocKind::Fn, type_def_id)
+                {
+                    self.tcx
+                        .generics_of(assoc_fn.def_id)
+                        .params
+                        .iter()
+                        .map(|param| match param.kind {
+                            ty::GenericParamDefKind::Lifetime => ty::BoundVariableKind::Region(
+                                ty::BoundRegionKind::BrNamed(param.def_id, param.name),
+                            ),
+                            ty::GenericParamDefKind::Type { .. } => ty::BoundVariableKind::Ty(
+                                ty::BoundTyKind::Param(param.def_id, param.name),
+                            ),
+                            ty::GenericParamDefKind::Const { .. } => ty::BoundVariableKind::Const,
+                        })
+                        .chain(self.tcx.fn_sig(assoc_fn.def_id).subst_identity().bound_vars())
+                        .collect()
+                } else {
+                    self.tcx.sess.delay_span_bug(
+                        binding.ident.span,
+                        "bad return type notation here",
+                    );
+                    vec![]
+                };
+                self.with(scope, |this| {
+                    let scope = Scope::Supertrait { bound_vars, s: this.scope };
+                    this.with(scope, |this| {
+                        let (bound_vars, _) = this.poly_trait_ref_binder_info();
+                        this.record_late_bound_vars(binding.hir_id, bound_vars);
+                        this.visit_assoc_type_binding(binding)
+                    });
+                });
+            } else if let Some(type_def_id) = type_def_id {
                 let bound_vars =
                     BoundVarContext::supertrait_hrtb_vars(self.tcx, type_def_id, binding.ident);
                 self.with(scope, |this| {
@@ -1698,8 +1749,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
             if trait_defines_associated_type_named(def_id) {
                 break Some(bound_vars.into_iter().collect());
             }
-            let predicates =
-                tcx.super_predicates_that_define_assoc_type((def_id, Some(assoc_name)));
+            let predicates = tcx.super_predicates_that_define_assoc_type((def_id, assoc_name));
             let obligations = predicates.predicates.iter().filter_map(|&(pred, _)| {
                 let bound_predicate = pred.kind();
                 match bound_predicate.skip_binder() {
diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs
index 50073d94e..c173bd913 100644
--- a/compiler/rustc_hir_analysis/src/collect/type_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs
@@ -8,9 +8,7 @@ use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::print::with_forced_trimmed_paths;
 use rustc_middle::ty::subst::InternalSubsts;
 use rustc_middle::ty::util::IntTypeExt;
-use rustc_middle::ty::{
-    self, DefIdTree, IsSuggestable, Ty, TyCtxt, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
-};
+use rustc_middle::ty::{self, ImplTraitInTraitData, IsSuggestable, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::symbol::Ident;
 use rustc_span::{Span, DUMMY_SP};
 
@@ -62,7 +60,7 @@ pub(super) fn opt_const_param_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<
                 .find(|(_, node)| matches!(node, OwnerNode::Item(_)))
                 .unwrap()
                 .0
-                .to_def_id();
+                .def_id;
             let item_ctxt = &ItemCtxt::new(tcx, item_def_id) as &dyn crate::astconv::AstConv<'_>;
             let ty = item_ctxt.ast_ty_to_ty(hir_ty);
 
@@ -243,24 +241,46 @@ fn get_path_containing_arg_in_pat<'hir>(
     arg_path
 }
 
-pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>> {
-    let def_id = def_id.expect_local();
+pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<Ty<'_>> {
+    // If we are computing `type_of` the synthesized associated type for an RPITIT in the impl
+    // side, use `collect_return_position_impl_trait_in_trait_tys` to infer the value of the
+    // associated type in the impl.
+    if let Some(ImplTraitInTraitData::Impl { fn_def_id, .. }) =
+        tcx.opt_rpitit_info(def_id.to_def_id())
+    {
+        match tcx.collect_return_position_impl_trait_in_trait_tys(fn_def_id) {
+            Ok(map) => {
+                let assoc_item = tcx.associated_item(def_id);
+                return ty::EarlyBinder(map[&assoc_item.trait_item_def_id.unwrap()]);
+            }
+            Err(_) => {
+                return ty::EarlyBinder(tcx.ty_error_with_message(
+                    DUMMY_SP,
+                    "Could not collect return position impl trait in trait tys",
+                ));
+            }
+        }
+    }
+
     use rustc_hir::*;
 
     let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
 
-    let icx = ItemCtxt::new(tcx, def_id.to_def_id());
+    let icx = ItemCtxt::new(tcx, def_id);
 
     let output = match tcx.hir().get(hir_id) {
         Node::TraitItem(item) => match item.kind {
             TraitItemKind::Fn(..) => {
-                let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
+                let substs = InternalSubsts::identity_for_item(tcx, def_id);
                 tcx.mk_fn_def(def_id.to_def_id(), substs)
             }
             TraitItemKind::Const(ty, body_id) => body_id
                 .and_then(|body_id| {
-                    is_suggestable_infer_ty(ty)
-                        .then(|| infer_placeholder_type(tcx, def_id, body_id, ty.span, item.ident, "constant",))
+                    is_suggestable_infer_ty(ty).then(|| {
+                        infer_placeholder_type(
+                            tcx, def_id, body_id, ty.span, item.ident, "constant",
+                        )
+                    })
                 })
                 .unwrap_or_else(|| icx.to_ty(ty)),
             TraitItemKind::Type(_, Some(ty)) => icx.to_ty(ty),
@@ -271,7 +291,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
 
         Node::ImplItem(item) => match item.kind {
             ImplItemKind::Fn(..) => {
-                let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
+                let substs = InternalSubsts::identity_for_item(tcx, def_id);
                 tcx.mk_fn_def(def_id.to_def_id(), substs)
             }
             ImplItemKind::Const(ty, body_id) => {
@@ -316,22 +336,23 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
                     }
                 }
                 ItemKind::TyAlias(self_ty, _) => icx.to_ty(self_ty),
-                ItemKind::Impl(hir::Impl { self_ty, .. }) => {
-                    match self_ty.find_self_aliases() {
-                        spans if spans.len() > 0 => {
-                            let guar = tcx.sess.emit_err(crate::errors::SelfInImplSelf { span: spans.into(), note: () });
-                            tcx.ty_error(guar)
-                        },
-                        _ => icx.to_ty(*self_ty),
+                ItemKind::Impl(hir::Impl { self_ty, .. }) => match self_ty.find_self_aliases() {
+                    spans if spans.len() > 0 => {
+                        let guar = tcx.sess.emit_err(crate::errors::SelfInImplSelf {
+                            span: spans.into(),
+                            note: (),
+                        });
+                        tcx.ty_error(guar)
                     }
+                    _ => icx.to_ty(*self_ty),
                 },
                 ItemKind::Fn(..) => {
-                    let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
+                    let substs = InternalSubsts::identity_for_item(tcx, def_id);
                     tcx.mk_fn_def(def_id.to_def_id(), substs)
                 }
                 ItemKind::Enum(..) | ItemKind::Struct(..) | ItemKind::Union(..) => {
                     let def = tcx.adt_def(def_id);
-                    let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
+                    let substs = InternalSubsts::identity_for_item(tcx, def_id);
                     tcx.mk_adt(def, substs)
                 }
                 ItemKind::OpaqueTy(OpaqueTy { origin: hir::OpaqueTyOrigin::TyAlias, .. }) => {
@@ -344,8 +365,11 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
                     in_trait,
                     ..
                 }) => {
-                    if in_trait {
-                        assert!(tcx.impl_defaultness(owner).has_value());
+                    if in_trait && !tcx.impl_defaultness(owner).has_value() {
+                        span_bug!(
+                            tcx.def_span(def_id),
+                            "tried to get type of this RPITIT with no definition"
+                        );
                     }
                     find_opaque_ty_constraints_for_rpit(tcx, def_id, owner)
                 }
@@ -368,7 +392,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
 
         Node::ForeignItem(foreign_item) => match foreign_item.kind {
             ForeignItemKind::Fn(..) => {
-                let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
+                let substs = InternalSubsts::identity_for_item(tcx, def_id);
                 tcx.mk_fn_def(def_id.to_def_id(), substs)
             }
             ForeignItemKind::Static(t, _) => icx.to_ty(t),
@@ -380,7 +404,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
                 tcx.type_of(tcx.hir().get_parent_item(hir_id)).subst_identity()
             }
             VariantData::Tuple(..) => {
-                let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
+                let substs = InternalSubsts::identity_for_item(tcx, def_id);
                 tcx.mk_fn_def(def_id.to_def_id(), substs)
             }
         },
@@ -413,7 +437,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
                 Node::Expr(Expr { kind: ExprKind::ConstBlock(anon_const), .. })
                     if anon_const.hir_id == hir_id =>
                 {
-                    let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
+                    let substs = InternalSubsts::identity_for_item(tcx, def_id);
                     substs.as_inline_const().ty()
                 }
 
@@ -434,15 +458,12 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
                     tcx.adt_def(tcx.hir().get_parent_item(hir_id)).repr().discr_type().to_ty(tcx)
                 }
 
-                Node::TypeBinding(
-                    TypeBinding {
-                        hir_id: binding_id,
-                        kind: TypeBindingKind::Equality { term: Term::Const(e) },
-                        ident,
-                        ..
-                    },
-                ) if let Node::TraitRef(trait_ref) =
-                    tcx.hir().get_parent(*binding_id)
+                Node::TypeBinding(TypeBinding {
+                    hir_id: binding_id,
+                    kind: TypeBindingKind::Equality { term: Term::Const(e) },
+                    ident,
+                    ..
+                }) if let Node::TraitRef(trait_ref) = tcx.hir().get_parent(*binding_id)
                     && e.hir_id == hir_id =>
                 {
                     let Some(trait_def_id) = trait_ref.trait_def_id() else {
@@ -456,7 +477,9 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
                         def_id.to_def_id(),
                     );
                     if let Some(assoc_item) = assoc_item {
-                        tcx.type_of(assoc_item.def_id).subst_identity()
+                        tcx.type_of(assoc_item.def_id)
+                            .no_bound_vars()
+                            .expect("const parameter types cannot be generic")
                     } else {
                         // FIXME(associated_const_equality): add a useful error message here.
                         tcx.ty_error_with_message(
@@ -466,10 +489,13 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
                     }
                 }
 
-                Node::TypeBinding(
-                    TypeBinding { hir_id: binding_id, gen_args, kind, ident, .. },
-                ) if let Node::TraitRef(trait_ref) =
-                    tcx.hir().get_parent(*binding_id)
+                Node::TypeBinding(TypeBinding {
+                    hir_id: binding_id,
+                    gen_args,
+                    kind,
+                    ident,
+                    ..
+                }) if let Node::TraitRef(trait_ref) = tcx.hir().get_parent(*binding_id)
                     && let Some((idx, _)) =
                         gen_args.args.iter().enumerate().find(|(_, arg)| {
                             if let GenericArg::Const(ct) = arg {
@@ -498,15 +524,18 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>>
                         },
                         def_id.to_def_id(),
                     );
-                    if let Some(param)
-                        = assoc_item.map(|item| &tcx.generics_of(item.def_id).params[idx]).filter(|param| param.kind.is_ty_or_const())
+                    if let Some(assoc_item) = assoc_item
+                        && let param = &tcx.generics_of(assoc_item.def_id).params[idx]
+                        && matches!(param.kind, ty::GenericParamDefKind::Const { .. })
                     {
-                        tcx.type_of(param.def_id).subst_identity()
+                        tcx.type_of(param.def_id)
+                            .no_bound_vars()
+                            .expect("const parameter types cannot be generic")
                     } else {
                         // FIXME(associated_const_equality): add a useful error message here.
                         tcx.ty_error_with_message(
                             DUMMY_SP,
-                            "Could not find associated const on trait",
+                            "Could not find const param on associated item",
                         )
                     }
                 }
@@ -574,7 +603,7 @@ fn find_opaque_ty_constraints_for_tait(tcx: TyCtxt<'_>, def_id: LocalDefId) -> T
         found: Option<ty::OpaqueHiddenType<'tcx>>,
 
         /// In the presence of dead code, typeck may figure out a hidden type
-        /// while borrowck will now. We collect these cases here and check at
+        /// while borrowck will not. We collect these cases here and check at
         /// the end that we actually found a type that matches (modulo regions).
         typeck_types: Vec<ty::OpaqueHiddenType<'tcx>>,
     }
@@ -746,7 +775,7 @@ fn find_opaque_ty_constraints_for_rpit(
             // Use borrowck to get the type with unerased regions.
             let concrete_opaque_types = &self.tcx.mir_borrowck(def_id).concrete_opaque_types;
             debug!(?concrete_opaque_types);
-            for &(def_id, concrete_type) in concrete_opaque_types {
+            for (&def_id, &concrete_type) in concrete_opaque_types {
                 if def_id != self.def_id {
                     // Ignore constraints for other opaque types.
                     continue;
@@ -842,28 +871,6 @@ fn infer_placeholder_type<'a>(
     item_ident: Ident,
     kind: &'static str,
 ) -> Ty<'a> {
-    // Attempts to make the type nameable by turning FnDefs into FnPtrs.
-    struct MakeNameable<'tcx> {
-        tcx: TyCtxt<'tcx>,
-    }
-
-    impl<'tcx> TypeFolder<TyCtxt<'tcx>> for MakeNameable<'tcx> {
-        fn interner(&self) -> TyCtxt<'tcx> {
-            self.tcx
-        }
-
-        fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
-            let ty = match *ty.kind() {
-                ty::FnDef(def_id, substs) => {
-                    self.tcx.mk_fn_ptr(self.tcx.fn_sig(def_id).subst(self.tcx, substs))
-                }
-                _ => ty,
-            };
-
-            ty.super_fold_with(self)
-        }
-    }
-
     let ty = tcx.diagnostic_only_typeck(def_id).node_type(body_id.hir_id);
 
     // If this came from a free `const` or `static mut?` item,
diff --git a/compiler/rustc_hir_analysis/src/errors.rs b/compiler/rustc_hir_analysis/src/errors.rs
index 3e0692757..2a3a68348 100644
--- a/compiler/rustc_hir_analysis/src/errors.rs
+++ b/compiler/rustc_hir_analysis/src/errors.rs
@@ -5,7 +5,7 @@ use rustc_errors::{
     error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
     MultiSpan,
 };
-use rustc_macros::Diagnostic;
+use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_middle::ty::Ty;
 use rustc_span::{symbol::Ident, Span, Symbol};
 
@@ -129,6 +129,18 @@ pub struct AssocTypeBindingNotAllowed {
     #[primary_span]
     #[label]
     pub span: Span,
+
+    #[subdiagnostic]
+    pub fn_trait_expansion: Option<ParenthesizedFnTraitExpansion>,
+}
+
+#[derive(Subdiagnostic)]
+#[help(hir_analysis_parenthesized_fn_trait_expansion)]
+pub struct ParenthesizedFnTraitExpansion {
+    #[primary_span]
+    pub span: Span,
+
+    pub expanded_type: String,
 }
 
 #[derive(Diagnostic)]
@@ -316,6 +328,14 @@ pub(crate) struct TrackCallerOnMain {
 }
 
 #[derive(Diagnostic)]
+#[diag(hir_analysis_target_feature_on_main)]
+pub(crate) struct TargetFeatureOnMain {
+    #[primary_span]
+    #[label(hir_analysis_target_feature_on_main)]
+    pub main: Span,
+}
+
+#[derive(Diagnostic)]
 #[diag(hir_analysis_start_not_track_caller)]
 pub(crate) struct StartTrackCaller {
     #[primary_span]
@@ -325,6 +345,15 @@ pub(crate) struct StartTrackCaller {
 }
 
 #[derive(Diagnostic)]
+#[diag(hir_analysis_start_not_target_feature)]
+pub(crate) struct StartTargetFeature {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub start: Span,
+}
+
+#[derive(Diagnostic)]
 #[diag(hir_analysis_start_not_async, code = "E0752")]
 pub(crate) struct StartAsync {
     #[primary_span]
@@ -399,3 +428,206 @@ pub(crate) enum CannotCaptureLateBoundInAnonConst {
         def_span: Span,
     },
 }
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_variances_of)]
+pub(crate) struct VariancesOf {
+    #[primary_span]
+    pub span: Span,
+    pub variances_of: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_pass_to_variadic_function, code = "E0617")]
+pub(crate) struct PassToVariadicFunction<'tcx, 'a> {
+    #[primary_span]
+    pub span: Span,
+    pub ty: Ty<'tcx>,
+    pub cast_ty: &'a str,
+    #[suggestion(code = "{replace}", applicability = "machine-applicable")]
+    pub sugg_span: Option<Span>,
+    pub replace: String,
+    #[help]
+    pub help: Option<()>,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_cast_thin_pointer_to_fat_pointer, code = "E0607")]
+pub(crate) struct CastThinPointerToFatPointer<'tcx> {
+    #[primary_span]
+    pub span: Span,
+    pub expr_ty: Ty<'tcx>,
+    pub cast_ty: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_invalid_union_field, code = "E0740")]
+pub(crate) struct InvalidUnionField {
+    #[primary_span]
+    pub field_span: Span,
+    #[subdiagnostic]
+    pub sugg: InvalidUnionFieldSuggestion,
+    #[note]
+    pub note: (),
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_return_type_notation_on_non_rpitit)]
+pub(crate) struct ReturnTypeNotationOnNonRpitit<'tcx> {
+    #[primary_span]
+    pub span: Span,
+    pub ty: Ty<'tcx>,
+    #[label]
+    pub fn_span: Option<Span>,
+    #[note]
+    pub note: (),
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(hir_analysis_invalid_union_field_sugg, applicability = "machine-applicable")]
+pub(crate) struct InvalidUnionFieldSuggestion {
+    #[suggestion_part(code = "std::mem::ManuallyDrop<")]
+    pub lo: Span,
+    #[suggestion_part(code = ">")]
+    pub hi: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_return_type_notation_equality_bound)]
+pub(crate) struct ReturnTypeNotationEqualityBound {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_return_type_notation_missing_method)]
+pub(crate) struct ReturnTypeNotationMissingMethod {
+    #[primary_span]
+    pub span: Span,
+    pub trait_name: Symbol,
+    pub assoc_name: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_placeholder_not_allowed_item_signatures, code = "E0121")]
+pub(crate) struct PlaceholderNotAllowedItemSignatures {
+    #[primary_span]
+    #[label]
+    pub spans: Vec<Span>,
+    pub kind: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_associated_type_trait_uninferred_generic_params, code = "E0212")]
+pub(crate) struct AssociatedTypeTraitUninferredGenericParams {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(style = "verbose", applicability = "maybe-incorrect", code = "{bound}")]
+    pub inferred_sugg: Option<Span>,
+    pub bound: String,
+    #[subdiagnostic]
+    pub mpart_sugg: Option<AssociatedTypeTraitUninferredGenericParamsMultipartSuggestion>,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    hir_analysis_associated_type_trait_uninferred_generic_params_multipart_suggestion,
+    applicability = "maybe-incorrect"
+)]
+pub(crate) struct AssociatedTypeTraitUninferredGenericParamsMultipartSuggestion {
+    #[suggestion_part(code = "{first}")]
+    pub fspan: Span,
+    pub first: String,
+    #[suggestion_part(code = "{second}")]
+    pub sspan: Span,
+    pub second: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_enum_discriminant_overflowed, code = "E0370")]
+#[note]
+pub(crate) struct EnumDiscriminantOverflowed {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub discr: String,
+    pub item_name: Symbol,
+    pub wrapped_discr: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_paren_sugar_attribute)]
+#[help]
+pub(crate) struct ParenSugarAttribute {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_must_implement_one_of_attribute)]
+pub(crate) struct MustImplementOneOfAttribute {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_must_be_name_of_associated_function)]
+pub(crate) struct MustBeNameOfAssociatedFunction {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_function_not_have_default_implementation)]
+pub(crate) struct FunctionNotHaveDefaultImplementation {
+    #[primary_span]
+    pub span: Span,
+    #[note]
+    pub note_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_must_implement_not_function)]
+pub(crate) struct MustImplementNotFunction {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub span_note: MustImplementNotFunctionSpanNote,
+    #[subdiagnostic]
+    pub note: MustImplementNotFunctionNote,
+}
+
+#[derive(Subdiagnostic)]
+#[note(hir_analysis_must_implement_not_function_span_note)]
+pub(crate) struct MustImplementNotFunctionSpanNote {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Subdiagnostic)]
+#[note(hir_analysis_must_implement_not_function_note)]
+pub(crate) struct MustImplementNotFunctionNote {}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_function_not_found_in_trait)]
+pub(crate) struct FunctionNotFoundInTrait {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_functions_names_duplicated)]
+#[note]
+pub(crate) struct FunctionNamesDuplicated {
+    #[primary_span]
+    pub spans: Vec<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_simd_ffi_highly_experimental)]
+#[help]
+pub(crate) struct SIMDFFIHighlyExperimental {
+    #[primary_span]
+    pub span: Span,
+    pub snip: String,
+}
diff --git a/compiler/rustc_hir_analysis/src/hir_wf_check.rs b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
index e330fcc78..8269a6dde 100644
--- a/compiler/rustc_hir_analysis/src/hir_wf_check.rs
+++ b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
@@ -7,7 +7,7 @@ use rustc_infer::traits::{ObligationCause, WellFormedLoc};
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, Region, TyCtxt, TypeFoldable, TypeFolder};
 use rustc_span::def_id::LocalDefId;
-use rustc_trait_selection::traits;
+use rustc_trait_selection::traits::{self, ObligationCtxt};
 
 pub fn provide(providers: &mut Providers) {
     *providers = Providers { diagnostic_hir_wf_check, ..*providers };
@@ -31,7 +31,7 @@ fn diagnostic_hir_wf_check<'tcx>(
     tcx.sess
         .delay_span_bug(tcx.def_span(def_id), "Performed HIR wfcheck without an existing error!");
 
-    let icx = ItemCtxt::new(tcx, def_id.to_def_id());
+    let icx = ItemCtxt::new(tcx, def_id);
 
     // To perform HIR-based WF checking, we iterate over all HIR types
     // that occur 'inside' the item we're checking. For example,
@@ -66,35 +66,35 @@ fn diagnostic_hir_wf_check<'tcx>(
     impl<'tcx> Visitor<'tcx> for HirWfCheck<'tcx> {
         fn visit_ty(&mut self, ty: &'tcx hir::Ty<'tcx>) {
             let infcx = self.tcx.infer_ctxt().build();
+            let ocx = ObligationCtxt::new(&infcx);
+
             let tcx_ty = self.icx.to_ty(ty).fold_with(&mut EraseAllBoundRegions { tcx: self.tcx });
             let cause = traits::ObligationCause::new(
                 ty.span,
                 self.def_id,
                 traits::ObligationCauseCode::WellFormed(None),
             );
-            let errors = traits::fully_solve_obligation(
-                &infcx,
-                traits::Obligation::new(
-                    self.tcx,
-                    cause,
-                    self.param_env,
-                    ty::Binder::dummy(ty::PredicateKind::WellFormed(tcx_ty.into())),
-                ),
-            );
-            if !errors.is_empty() {
-                debug!("Wf-check got errors for {:?}: {:?}", ty, errors);
-                for error in errors {
-                    if error.obligation.predicate == self.predicate {
-                        // Save the cause from the greatest depth - this corresponds
-                        // to picking more-specific types (e.g. `MyStruct<u8>`)
-                        // over less-specific types (e.g. `Option<MyStruct<u8>>`)
-                        if self.depth >= self.cause_depth {
-                            self.cause = Some(error.obligation.cause);
-                            self.cause_depth = self.depth
-                        }
+
+            ocx.register_obligation(traits::Obligation::new(
+                self.tcx,
+                cause,
+                self.param_env,
+                ty::PredicateKind::WellFormed(tcx_ty.into()),
+            ));
+
+            for error in ocx.select_all_or_error() {
+                debug!("Wf-check got error for {:?}: {:?}", ty, error);
+                if error.obligation.predicate == self.predicate {
+                    // Save the cause from the greatest depth - this corresponds
+                    // to picking more-specific types (e.g. `MyStruct<u8>`)
+                    // over less-specific types (e.g. `Option<MyStruct<u8>>`)
+                    if self.depth >= self.cause_depth {
+                        self.cause = Some(error.obligation.cause);
+                        self.cause_depth = self.depth
                     }
                 }
             }
+
             self.depth += 1;
             intravisit::walk_ty(self, ty);
             self.depth -= 1;
diff --git a/compiler/rustc_hir_analysis/src/impl_wf_check/min_specialization.rs b/compiler/rustc_hir_analysis/src/impl_wf_check/min_specialization.rs
index daa5d1570..eb2fc3952 100644
--- a/compiler/rustc_hir_analysis/src/impl_wf_check/min_specialization.rs
+++ b/compiler/rustc_hir_analysis/src/impl_wf_check/min_specialization.rs
@@ -168,20 +168,19 @@ fn get_impl_substs(
     let assumed_wf_types =
         ocx.assumed_wf_types(param_env, tcx.def_span(impl1_def_id), impl1_def_id);
 
-    let impl1_substs = InternalSubsts::identity_for_item(tcx, impl1_def_id.to_def_id());
+    let impl1_substs = InternalSubsts::identity_for_item(tcx, impl1_def_id);
     let impl2_substs =
         translate_substs(infcx, param_env, impl1_def_id.to_def_id(), impl1_substs, impl2_node);
 
     let errors = ocx.select_all_or_error();
     if !errors.is_empty() {
-        ocx.infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+        ocx.infcx.err_ctxt().report_fulfillment_errors(&errors);
         return None;
     }
 
     let implied_bounds = infcx.implied_bounds_tys(param_env, impl1_def_id, assumed_wf_types);
-    let outlives_env = OutlivesEnvironment::with_bounds(param_env, Some(infcx), implied_bounds);
-    let _ =
-        infcx.err_ctxt().check_region_obligations_and_report_errors(impl1_def_id, &outlives_env);
+    let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds);
+    let _ = ocx.resolve_regions_and_report_errors(impl1_def_id, &outlives_env);
     let Ok(impl2_substs) = infcx.fully_resolve(impl2_substs) else {
         let span = tcx.def_span(impl1_def_id);
         tcx.sess.emit_err(SubstsOnOverriddenImpl { span });
@@ -318,30 +317,20 @@ fn check_predicates<'tcx>(
     span: Span,
 ) {
     let instantiated = tcx.predicates_of(impl1_def_id).instantiate(tcx, impl1_substs);
-    let impl1_predicates: Vec<_> = traits::elaborate_predicates_with_span(
-        tcx,
-        std::iter::zip(
-            instantiated.predicates,
-            // Don't drop predicates (unsound!) because `spans` is too short
-            instantiated.spans.into_iter().chain(std::iter::repeat(span)),
-        ),
-    )
-    .map(|obligation| (obligation.predicate, obligation.cause.span))
-    .collect();
+    let impl1_predicates: Vec<_> = traits::elaborate(tcx, instantiated.into_iter()).collect();
 
     let mut impl2_predicates = if impl2_node.is_from_trait() {
         // Always applicable traits have to be always applicable without any
         // assumptions.
         Vec::new()
     } else {
-        traits::elaborate_predicates(
+        traits::elaborate(
             tcx,
             tcx.predicates_of(impl2_node.def_id())
                 .instantiate(tcx, impl2_substs)
                 .predicates
                 .into_iter(),
         )
-        .map(|obligation| obligation.predicate)
         .collect()
     };
     debug!(?impl1_predicates, ?impl2_predicates);
@@ -361,12 +350,16 @@ fn check_predicates<'tcx>(
     // which is sound because we forbid impls like the following
     //
     // impl<D: Debug> AlwaysApplicable for D { }
-    let always_applicable_traits = impl1_predicates.iter().copied().filter(|&(predicate, _)| {
-        matches!(
-            trait_predicate_kind(tcx, predicate),
-            Some(TraitSpecializationKind::AlwaysApplicable)
-        )
-    });
+    let always_applicable_traits = impl1_predicates
+        .iter()
+        .copied()
+        .filter(|&(predicate, _)| {
+            matches!(
+                trait_predicate_kind(tcx, predicate),
+                Some(TraitSpecializationKind::AlwaysApplicable)
+            )
+        })
+        .map(|(pred, _span)| pred);
 
     // Include the well-formed predicates of the type parameters of the impl.
     for arg in tcx.impl_trait_ref(impl1_def_id).unwrap().subst_identity().substs {
@@ -376,14 +369,10 @@ fn check_predicates<'tcx>(
                 .unwrap();
 
         assert!(!obligations.needs_infer());
-        impl2_predicates.extend(
-            traits::elaborate_obligations(tcx, obligations).map(|obligation| obligation.predicate),
-        )
+        impl2_predicates
+            .extend(traits::elaborate(tcx, obligations).map(|obligation| obligation.predicate))
     }
-    impl2_predicates.extend(
-        traits::elaborate_predicates_with_span(tcx, always_applicable_traits)
-            .map(|obligation| obligation.predicate),
-    );
+    impl2_predicates.extend(traits::elaborate(tcx, always_applicable_traits));
 
     for (predicate, span) in impl1_predicates {
         if !impl2_predicates.iter().any(|pred2| trait_predicates_eq(tcx, predicate, *pred2, span)) {
@@ -528,7 +517,7 @@ fn trait_predicate_kind<'tcx>(
         | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(_))
         | ty::PredicateKind::Clause(ty::Clause::Projection(_))
         | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
-        | ty::PredicateKind::AliasEq(..)
+        | ty::PredicateKind::AliasRelate(..)
         | ty::PredicateKind::WellFormed(_)
         | ty::PredicateKind::Subtype(_)
         | ty::PredicateKind::Coerce(_)
diff --git a/compiler/rustc_hir_analysis/src/lib.rs b/compiler/rustc_hir_analysis/src/lib.rs
index 33c132fd5..27e561803 100644
--- a/compiler/rustc_hir_analysis/src/lib.rs
+++ b/compiler/rustc_hir_analysis/src/lib.rs
@@ -67,10 +67,9 @@ This API is completely unstable and subject to change.
 #![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(never_type)]
-#![feature(once_cell)]
+#![feature(lazy_cell)]
 #![feature(slice_partition_dedup)]
 #![feature(try_blocks)]
-#![feature(is_some_and)]
 #![feature(type_alias_impl_trait)]
 #![recursion_limit = "256"]
 
@@ -102,7 +101,7 @@ use rustc_errors::ErrorGuaranteed;
 use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir as hir;
 use rustc_hir::Node;
-use rustc_infer::infer::{InferOk, TyCtxtInferExt};
+use rustc_infer::infer::TyCtxtInferExt;
 use rustc_macros::fluent_messages;
 use rustc_middle::middle;
 use rustc_middle::ty::query::Providers;
@@ -113,14 +112,14 @@ use rustc_span::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
 use rustc_span::{symbol::sym, Span, DUMMY_SP};
 use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
-use rustc_trait_selection::traits::{self, ObligationCause, ObligationCauseCode};
+use rustc_trait_selection::traits::{self, ObligationCause, ObligationCauseCode, ObligationCtxt};
 
 use std::ops::Not;
 
 use astconv::AstConv;
 use bounds::Bounds;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 fn require_c_abi_if_c_variadic(tcx: TyCtxt<'_>, decl: &hir::FnDecl<'_>, abi: Abi, span: Span) {
     const CONVENTIONS_UNSTABLE: &str = "`C`, `cdecl`, `win64`, `sysv64` or `efiapi`";
@@ -160,24 +159,21 @@ fn require_c_abi_if_c_variadic(tcx: TyCtxt<'_>, decl: &hir::FnDecl<'_>, abi: Abi
 fn require_same_types<'tcx>(
     tcx: TyCtxt<'tcx>,
     cause: &ObligationCause<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
     expected: Ty<'tcx>,
     actual: Ty<'tcx>,
-) -> bool {
+) {
     let infcx = &tcx.infer_ctxt().build();
-    let param_env = ty::ParamEnv::empty();
-    let errors = match infcx.at(cause, param_env).eq(expected, actual) {
-        Ok(InferOk { obligations, .. }) => traits::fully_solve_obligations(infcx, obligations),
+    let ocx = ObligationCtxt::new(infcx);
+    match ocx.eq(cause, param_env, expected, actual) {
+        Ok(()) => {
+            let errors = ocx.select_all_or_error();
+            if !errors.is_empty() {
+                infcx.err_ctxt().report_fulfillment_errors(&errors);
+            }
+        }
         Err(err) => {
             infcx.err_ctxt().report_mismatched_types(cause, expected, actual, err).emit();
-            return false;
-        }
-    };
-
-    match &errors[..] {
-        [] => true,
-        errors => {
-            infcx.err_ctxt().report_fulfillment_errors(errors, None);
-            false
         }
     }
 }
@@ -283,10 +279,21 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         error = true;
     }
 
+    if !tcx.codegen_fn_attrs(main_def_id).target_features.is_empty()
+        // Calling functions with `#[target_feature]` is not unsafe on WASM, see #84988
+        && !tcx.sess.target.is_like_wasm
+        && !tcx.sess.opts.actually_rustdoc
+    {
+        tcx.sess.emit_err(errors::TargetFeatureOnMain { main: main_span });
+        error = true;
+    }
+
     if error {
         return;
     }
 
+    // Main should have no WC, so empty param env is OK here.
+    let param_env = ty::ParamEnv::empty();
     let expected_return_type;
     if let Some(term_did) = tcx.lang_items().termination() {
         let return_ty = main_fnsig.output();
@@ -297,8 +304,6 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         }
         let return_ty = return_ty.skip_binder();
         let infcx = tcx.infer_ctxt().build();
-        // Main should have no WC, so empty param env is OK here.
-        let param_env = ty::ParamEnv::empty();
         let cause = traits::ObligationCause::new(
             return_ty_span,
             main_diagnostics_def_id,
@@ -309,7 +314,7 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         ocx.register_bound(cause, param_env, norm_return_ty, term_did);
         let errors = ocx.select_all_or_error();
         if !errors.is_empty() {
-            infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+            infcx.err_ctxt().report_fulfillment_errors(&errors);
             error = true;
         }
         // now we can take the return type of the given main function
@@ -334,6 +339,7 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
             main_diagnostics_def_id,
             ObligationCauseCode::MainFunctionType,
         ),
+        param_env,
         se_ty,
         tcx.mk_fn_ptr(main_fnsig),
     );
@@ -373,6 +379,18 @@ fn check_start_fn_ty(tcx: TyCtxt<'_>, start_def_id: DefId) {
                             });
                             error = true;
                         }
+                        if attr.has_name(sym::target_feature)
+                            // Calling functions with `#[target_feature]` is
+                            // not unsafe on WASM, see #84988
+                            && !tcx.sess.target.is_like_wasm
+                            && !tcx.sess.opts.actually_rustdoc
+                        {
+                            tcx.sess.emit_err(errors::StartTargetFeature {
+                                span: attr.span,
+                                start: start_span,
+                            });
+                            error = true;
+                        }
                     }
 
                     if error {
@@ -396,6 +414,7 @@ fn check_start_fn_ty(tcx: TyCtxt<'_>, start_def_id: DefId) {
                     start_def_id,
                     ObligationCauseCode::StartFunctionType,
                 ),
+                ty::ParamEnv::empty(), // start should not have any where bounds.
                 se_ty,
                 tcx.mk_fn_ptr(tcx.fn_sig(start_def_id).subst_identity()),
             );
@@ -492,7 +511,7 @@ pub fn hir_ty_to_ty<'tcx>(tcx: TyCtxt<'tcx>, hir_ty: &hir::Ty<'_>) -> Ty<'tcx> {
     // def-ID that will be used to determine the traits/predicates in
     // scope. This is derived from the enclosing item-like thing.
     let env_def_id = tcx.hir().get_parent_item(hir_ty.hir_id);
-    let item_cx = self::collect::ItemCtxt::new(tcx, env_def_id.to_def_id());
+    let item_cx = self::collect::ItemCtxt::new(tcx, env_def_id.def_id);
     item_cx.astconv().ast_ty_to_ty(hir_ty)
 }
 
@@ -505,7 +524,7 @@ pub fn hir_trait_to_predicates<'tcx>(
     // def-ID that will be used to determine the traits/predicates in
     // scope. This is derived from the enclosing item-like thing.
     let env_def_id = tcx.hir().get_parent_item(hir_trait.hir_ref_id);
-    let item_cx = self::collect::ItemCtxt::new(tcx, env_def_id.to_def_id());
+    let item_cx = self::collect::ItemCtxt::new(tcx, env_def_id.def_id);
     let mut bounds = Bounds::default();
     let _ = &item_cx.astconv().instantiate_poly_trait_ref(
         hir_trait,
diff --git a/compiler/rustc_hir_analysis/src/outlives/explicit.rs b/compiler/rustc_hir_analysis/src/outlives/explicit.rs
index 9ee678597..357deb07b 100644
--- a/compiler/rustc_hir_analysis/src/outlives/explicit.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/explicit.rs
@@ -56,7 +56,7 @@ impl<'tcx> ExplicitPredicatesMap<'tcx> {
                     | ty::PredicateKind::Clause(ty::Clause::Projection(..))
                     | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
                     | ty::PredicateKind::WellFormed(..)
-                    | ty::PredicateKind::AliasEq(..)
+                    | ty::PredicateKind::AliasRelate(..)
                     | ty::PredicateKind::ObjectSafe(..)
                     | ty::PredicateKind::ClosureKind(..)
                     | ty::PredicateKind::Subtype(..)
diff --git a/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs
index a8b33c74b..d53c429ca 100644
--- a/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs
@@ -1,7 +1,7 @@
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
-use rustc_middle::ty::{self, DefIdTree, Ty, TyCtxt};
+use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::ty::{GenericArg, GenericArgKind};
 use rustc_span::Span;
 
diff --git a/compiler/rustc_hir_analysis/src/outlives/mod.rs b/compiler/rustc_hir_analysis/src/outlives/mod.rs
index 81fe32000..da72d2584 100644
--- a/compiler/rustc_hir_analysis/src/outlives/mod.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/mod.rs
@@ -1,6 +1,6 @@
 use hir::Node;
 use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::LocalDefId;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::subst::GenericArgKind;
 use rustc_middle::ty::{self, CratePredicatesMap, TyCtxt};
@@ -17,8 +17,8 @@ pub fn provide(providers: &mut Providers) {
     *providers = Providers { inferred_outlives_of, inferred_outlives_crate, ..*providers };
 }
 
-fn inferred_outlives_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[(ty::Clause<'_>, Span)] {
-    let id = tcx.hir().local_def_id_to_hir_id(item_def_id.expect_local());
+fn inferred_outlives_of(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[(ty::Clause<'_>, Span)] {
+    let id = tcx.hir().local_def_id_to_hir_id(item_def_id);
 
     if matches!(tcx.def_kind(item_def_id), hir::def::DefKind::AnonConst) && tcx.lazy_normalization()
     {
@@ -45,7 +45,8 @@ fn inferred_outlives_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[(ty::Clause<'_
             hir::ItemKind::Struct(..) | hir::ItemKind::Enum(..) | hir::ItemKind::Union(..) => {
                 let crate_map = tcx.inferred_outlives_crate(());
 
-                let predicates = crate_map.predicates.get(&item_def_id).copied().unwrap_or(&[]);
+                let predicates =
+                    crate_map.predicates.get(&item_def_id.to_def_id()).copied().unwrap_or(&[]);
 
                 if tcx.has_attr(item_def_id, sym::rustc_outlives) {
                     let mut pred: Vec<String> = predicates
diff --git a/compiler/rustc_hir_analysis/src/outlives/test.rs b/compiler/rustc_hir_analysis/src/outlives/test.rs
index fa2ac5659..60f8e246a 100644
--- a/compiler/rustc_hir_analysis/src/outlives/test.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/test.rs
@@ -6,7 +6,7 @@ pub fn test_inferred_outlives(tcx: TyCtxt<'_>) {
     for id in tcx.hir().items() {
         // For unit testing: check for a special "rustc_outlives"
         // attribute and report an error with various results if found.
-        if tcx.has_attr(id.owner_id.to_def_id(), sym::rustc_outlives) {
+        if tcx.has_attr(id.owner_id, sym::rustc_outlives) {
             let inferred_outlives_of = tcx.inferred_outlives_of(id.owner_id);
             struct_span_err!(
                 tcx.sess,
diff --git a/compiler/rustc_hir_analysis/src/structured_errors/missing_cast_for_variadic_arg.rs b/compiler/rustc_hir_analysis/src/structured_errors/missing_cast_for_variadic_arg.rs
index 089491bef..0bfbf99cb 100644
--- a/compiler/rustc_hir_analysis/src/structured_errors/missing_cast_for_variadic_arg.rs
+++ b/compiler/rustc_hir_analysis/src/structured_errors/missing_cast_for_variadic_arg.rs
@@ -1,5 +1,5 @@
-use crate::structured_errors::StructuredDiagnostic;
-use rustc_errors::{Applicability, DiagnosticBuilder, DiagnosticId, ErrorGuaranteed};
+use crate::{errors, structured_errors::StructuredDiagnostic};
+use rustc_errors::{DiagnosticBuilder, DiagnosticId, ErrorGuaranteed};
 use rustc_middle::ty::{Ty, TypeVisitableExt};
 use rustc_session::Session;
 use rustc_span::Span;
@@ -21,27 +21,26 @@ impl<'tcx> StructuredDiagnostic<'tcx> for MissingCastForVariadicArg<'tcx, '_> {
     }
 
     fn diagnostic_common(&self) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        let mut err = self.sess.struct_span_err_with_code(
-            self.span,
-            &format!("can't pass `{}` to variadic function", self.ty),
-            self.code(),
-        );
+        let (sugg_span, replace, help) =
+            if let Ok(snippet) = self.sess.source_map().span_to_snippet(self.span) {
+                (Some(self.span), format!("{} as {}", snippet, self.cast_ty), None)
+            } else {
+                (None, "".to_string(), Some(()))
+            };
+
+        let mut err = self.sess.create_err(errors::PassToVariadicFunction {
+            span: self.span,
+            ty: self.ty,
+            cast_ty: self.cast_ty,
+            help,
+            replace,
+            sugg_span,
+        });
 
         if self.ty.references_error() {
             err.downgrade_to_delayed_bug();
         }
 
-        if let Ok(snippet) = self.sess.source_map().span_to_snippet(self.span) {
-            err.span_suggestion(
-                self.span,
-                &format!("cast the value to `{}`", self.cast_ty),
-                format!("{} as {}", snippet, self.cast_ty),
-                Applicability::MachineApplicable,
-            );
-        } else {
-            err.help(&format!("cast the value to `{}`", self.cast_ty));
-        }
-
         err
     }
 
diff --git a/compiler/rustc_hir_analysis/src/structured_errors/sized_unsized_cast.rs b/compiler/rustc_hir_analysis/src/structured_errors/sized_unsized_cast.rs
index 3b9fb3678..910417abe 100644
--- a/compiler/rustc_hir_analysis/src/structured_errors/sized_unsized_cast.rs
+++ b/compiler/rustc_hir_analysis/src/structured_errors/sized_unsized_cast.rs
@@ -1,4 +1,4 @@
-use crate::structured_errors::StructuredDiagnostic;
+use crate::{errors, structured_errors::StructuredDiagnostic};
 use rustc_errors::{DiagnosticBuilder, DiagnosticId, ErrorGuaranteed};
 use rustc_middle::ty::{Ty, TypeVisitableExt};
 use rustc_session::Session;
@@ -21,14 +21,11 @@ impl<'tcx> StructuredDiagnostic<'tcx> for SizedUnsizedCast<'tcx> {
     }
 
     fn diagnostic_common(&self) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        let mut err = self.sess.struct_span_err_with_code(
-            self.span,
-            &format!(
-                "cannot cast thin pointer `{}` to fat pointer `{}`",
-                self.expr_ty, self.cast_ty
-            ),
-            self.code(),
-        );
+        let mut err = self.sess.create_err(errors::CastThinPointerToFatPointer {
+            span: self.span,
+            expr_ty: self.expr_ty,
+            cast_ty: self.cast_ty.to_owned(),
+        });
 
         if self.expr_ty.references_error() {
             err.downgrade_to_delayed_bug();
diff --git a/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs b/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs
index cae884ae8..8f4d81ec3 100644
--- a/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs
+++ b/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs
@@ -565,7 +565,7 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> {
     /// type Map = HashMap<String>;
     /// ```
     fn suggest_adding_args(&self, err: &mut Diagnostic) {
-        if self.gen_args.parenthesized {
+        if self.gen_args.parenthesized != hir::GenericArgsParentheses::No {
             return;
         }
 
@@ -962,7 +962,11 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> {
 
             let msg = format!(
                 "remove these {}generics",
-                if self.gen_args.parenthesized { "parenthetical " } else { "" },
+                if self.gen_args.parenthesized == hir::GenericArgsParentheses::ParenSugar {
+                    "parenthetical "
+                } else {
+                    ""
+                },
             );
 
             err.span_suggestion(span, &msg, "", Applicability::MaybeIncorrect);
diff --git a/compiler/rustc_hir_analysis/src/variance/mod.rs b/compiler/rustc_hir_analysis/src/variance/mod.rs
index 5d5c8ca60..0a45119ff 100644
--- a/compiler/rustc_hir_analysis/src/variance/mod.rs
+++ b/compiler/rustc_hir_analysis/src/variance/mod.rs
@@ -8,7 +8,7 @@ use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, CrateVariancesMap, SubstsRef, Ty, TyCtxt};
-use rustc_middle::ty::{DefIdTree, TypeSuperVisitable, TypeVisitable};
+use rustc_middle::ty::{TypeSuperVisitable, TypeVisitable};
 use std::ops::ControlFlow;
 
 /// Defines the `TermsContext` basically houses an arena where we can
@@ -38,7 +38,7 @@ fn crate_variances(tcx: TyCtxt<'_>, (): ()) -> CrateVariancesMap<'_> {
     solve::solve_constraints(constraints_cx)
 }
 
-fn variances_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[ty::Variance] {
+fn variances_of(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[ty::Variance] {
     // Skip items with no generics - there's nothing to infer in them.
     if tcx.generics_of(item_def_id).count() == 0 {
         return &[];
@@ -53,7 +53,7 @@ fn variances_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[ty::Variance] {
         | DefKind::Variant
         | DefKind::Ctor(..) => {}
         DefKind::OpaqueTy | DefKind::ImplTraitPlaceholder => {
-            return variance_of_opaque(tcx, item_def_id.expect_local());
+            return variance_of_opaque(tcx, item_def_id);
         }
         _ => {
             // Variance not relevant.
@@ -64,7 +64,7 @@ fn variances_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[ty::Variance] {
     // Everything else must be inferred.
 
     let crate_map = tcx.crate_variances(());
-    crate_map.variances.get(&item_def_id).copied().unwrap_or(&[])
+    crate_map.variances.get(&item_def_id.to_def_id()).copied().unwrap_or(&[])
 }
 
 #[instrument(level = "trace", skip(tcx), ret)]
@@ -112,10 +112,14 @@ fn variance_of_opaque(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[ty::Varianc
         fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
             match t.kind() {
                 ty::Alias(_, ty::AliasTy { def_id, substs, .. })
-                    if matches!(
-                        self.tcx.def_kind(*def_id),
-                        DefKind::OpaqueTy | DefKind::ImplTraitPlaceholder
-                    ) =>
+                    if matches!(self.tcx.def_kind(*def_id), DefKind::OpaqueTy) =>
+                {
+                    self.visit_opaque(*def_id, substs)
+                }
+                // FIXME(-Zlower-impl-trait-in-trait-to-assoc-ty) check whether this is necessary
+                // at all for RPITITs.
+                ty::Alias(_, ty::AliasTy { def_id, substs, .. })
+                    if self.tcx.is_impl_trait_in_trait(*def_id) =>
                 {
                     self.visit_opaque(*def_id, substs)
                 }
@@ -148,7 +152,7 @@ fn variance_of_opaque(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[ty::Varianc
 
     let mut collector =
         OpaqueTypeLifetimeCollector { tcx, root_def_id: item_def_id.to_def_id(), variances };
-    let id_substs = ty::InternalSubsts::identity_for_item(tcx, item_def_id.to_def_id());
+    let id_substs = ty::InternalSubsts::identity_for_item(tcx, item_def_id);
     for pred in tcx.bound_explicit_item_bounds(item_def_id.to_def_id()).transpose_iter() {
         let pred = pred.map_bound(|(pred, _)| *pred).subst(tcx, id_substs);
         debug!(?pred);
diff --git a/compiler/rustc_hir_analysis/src/variance/test.rs b/compiler/rustc_hir_analysis/src/variance/test.rs
index 5feeb92d3..d57d05d76 100644
--- a/compiler/rustc_hir_analysis/src/variance/test.rs
+++ b/compiler/rustc_hir_analysis/src/variance/test.rs
@@ -1,14 +1,19 @@
 use rustc_middle::ty::TyCtxt;
 use rustc_span::symbol::sym;
 
+use crate::errors;
+
 pub fn test_variance(tcx: TyCtxt<'_>) {
     // For unit testing: check for a special "rustc_variance"
     // attribute and report an error with various results if found.
     for id in tcx.hir().items() {
-        if tcx.has_attr(id.owner_id.to_def_id(), sym::rustc_variance) {
+        if tcx.has_attr(id.owner_id, sym::rustc_variance) {
             let variances_of = tcx.variances_of(id.owner_id);
 
-            tcx.sess.struct_span_err(tcx.def_span(id.owner_id), format!("{variances_of:?}")).emit();
+            tcx.sess.emit_err(errors::VariancesOf {
+                span: tcx.def_span(id.owner_id),
+                variances_of: format!("{variances_of:?}"),
+            });
         }
     }
 }
diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs
index c021fca71..74f5b3590 100644
--- a/compiler/rustc_hir_pretty/src/lib.rs
+++ b/compiler/rustc_hir_pretty/src/lib.rs
@@ -242,7 +242,7 @@ pub fn enum_def_to_string(
 impl<'a> State<'a> {
     pub fn bclose_maybe_open(&mut self, span: rustc_span::Span, close_box: bool) {
         self.maybe_print_comment(span.hi());
-        self.break_offset_if_not_bol(1, -(INDENT_UNIT as isize));
+        self.break_offset_if_not_bol(1, -INDENT_UNIT);
         self.word("}");
         if close_box {
             self.end(); // close the outer-box
@@ -1366,10 +1366,6 @@ impl<'a> State<'a> {
         self.ibox(INDENT_UNIT);
         self.ann.pre(self, AnnNode::Expr(expr));
         match expr.kind {
-            hir::ExprKind::Box(expr) => {
-                self.word_space("box");
-                self.print_expr_maybe_paren(expr, parser::PREC_PREFIX);
-            }
             hir::ExprKind::Array(exprs) => {
                 self.print_expr_vec(exprs);
             }
@@ -1656,61 +1652,65 @@ impl<'a> State<'a> {
         generic_args: &hir::GenericArgs<'_>,
         colons_before_params: bool,
     ) {
-        if generic_args.parenthesized {
-            self.word("(");
-            self.commasep(Inconsistent, generic_args.inputs(), |s, ty| s.print_type(ty));
-            self.word(")");
+        match generic_args.parenthesized {
+            hir::GenericArgsParentheses::No => {
+                let start = if colons_before_params { "::<" } else { "<" };
+                let empty = Cell::new(true);
+                let start_or_comma = |this: &mut Self| {
+                    if empty.get() {
+                        empty.set(false);
+                        this.word(start)
+                    } else {
+                        this.word_space(",")
+                    }
+                };
+
+                let mut nonelided_generic_args: bool = false;
+                let elide_lifetimes = generic_args.args.iter().all(|arg| match arg {
+                    GenericArg::Lifetime(lt) if lt.is_elided() => true,
+                    GenericArg::Lifetime(_) => {
+                        nonelided_generic_args = true;
+                        false
+                    }
+                    _ => {
+                        nonelided_generic_args = true;
+                        true
+                    }
+                });
 
-            self.space_if_not_bol();
-            self.word_space("->");
-            self.print_type(generic_args.bindings[0].ty());
-        } else {
-            let start = if colons_before_params { "::<" } else { "<" };
-            let empty = Cell::new(true);
-            let start_or_comma = |this: &mut Self| {
-                if empty.get() {
-                    empty.set(false);
-                    this.word(start)
-                } else {
-                    this.word_space(",")
+                if nonelided_generic_args {
+                    start_or_comma(self);
+                    self.commasep(Inconsistent, generic_args.args, |s, generic_arg| {
+                        match generic_arg {
+                            GenericArg::Lifetime(lt) if !elide_lifetimes => s.print_lifetime(lt),
+                            GenericArg::Lifetime(_) => {}
+                            GenericArg::Type(ty) => s.print_type(ty),
+                            GenericArg::Const(ct) => s.print_anon_const(&ct.value),
+                            GenericArg::Infer(_inf) => s.word("_"),
+                        }
+                    });
                 }
-            };
 
-            let mut nonelided_generic_args: bool = false;
-            let elide_lifetimes = generic_args.args.iter().all(|arg| match arg {
-                GenericArg::Lifetime(lt) if lt.is_elided() => true,
-                GenericArg::Lifetime(_) => {
-                    nonelided_generic_args = true;
-                    false
+                for binding in generic_args.bindings {
+                    start_or_comma(self);
+                    self.print_type_binding(binding);
                 }
-                _ => {
-                    nonelided_generic_args = true;
-                    true
-                }
-            });
-
-            if nonelided_generic_args {
-                start_or_comma(self);
-                self.commasep(
-                    Inconsistent,
-                    generic_args.args,
-                    |s, generic_arg| match generic_arg {
-                        GenericArg::Lifetime(lt) if !elide_lifetimes => s.print_lifetime(lt),
-                        GenericArg::Lifetime(_) => {}
-                        GenericArg::Type(ty) => s.print_type(ty),
-                        GenericArg::Const(ct) => s.print_anon_const(&ct.value),
-                        GenericArg::Infer(_inf) => s.word("_"),
-                    },
-                );
-            }
 
-            for binding in generic_args.bindings {
-                start_or_comma(self);
-                self.print_type_binding(binding);
+                if !empty.get() {
+                    self.word(">")
+                }
             }
+            hir::GenericArgsParentheses::ParenSugar => {
+                self.word("(");
+                self.commasep(Inconsistent, generic_args.inputs(), |s, ty| s.print_type(ty));
+                self.word(")");
 
-            if !empty.get() {
-                self.word(">")
+                self.space_if_not_bol();
+                self.word_space("->");
+                self.print_type(generic_args.bindings[0].ty());
+            }
+            hir::GenericArgsParentheses::ReturnTypeNotation => {
+                self.word("(..)");
             }
         }
     }
diff --git a/compiler/rustc_hir_typeck/locales/en-US.ftl b/compiler/rustc_hir_typeck/messages.ftl
index adfcbc36a..2c537bf40 100644
--- a/compiler/rustc_hir_typeck/locales/en-US.ftl
+++ b/compiler/rustc_hir_typeck/messages.ftl
@@ -4,14 +4,14 @@ hir_typeck_field_multiply_specified_in_initializer =
     .previous_use_label = first use of `{$ident}`
 
 hir_typeck_copy_impl_on_type_with_dtor =
-    the trait `Copy` may not be implemented for this type; the type has a destructor
+    the trait `Copy` cannot be implemented for this type; the type has a destructor
     .label = `Copy` not allowed on types with destructors
 
 hir_typeck_multiple_relaxed_default_bounds =
     type parameter has more than one relaxed default bound, only one is supported
 
 hir_typeck_copy_impl_on_non_adt =
-    the trait `Copy` may not be implemented for this type
+    the trait `Copy` cannot be implemented for this type
     .label = type is not a structure or enumeration
 
 hir_typeck_trait_object_declared_with_no_traits =
diff --git a/compiler/rustc_hir_typeck/src/_match.rs b/compiler/rustc_hir_typeck/src/_match.rs
index e19ef2ff3..6c2ce6272 100644
--- a/compiler/rustc_hir_typeck/src/_match.rs
+++ b/compiler/rustc_hir_typeck/src/_match.rs
@@ -299,7 +299,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             {
                 // check that the `if` expr without `else` is the fn body's expr
                 if expr.span == sp {
-                    return self.get_fn_decl(hir_id).and_then(|(fn_decl, _)| {
+                    return self.get_fn_decl(hir_id).and_then(|(_, fn_decl, _)| {
                         let span = fn_decl.output.span();
                         let snippet = self.tcx.sess.source_map().span_to_snippet(span).ok()?;
                         Some((span, format!("expected `{snippet}` because of this return type")))
@@ -538,8 +538,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 // FIXME(rpitit): This will need to be fixed when we move to associated types
                                 assert!(matches!(
                                     *trait_pred.trait_ref.self_ty().kind(),
-                                    ty::Alias(_, ty::AliasTy { def_id, substs, .. })
-                                    if def_id == rpit_def_id && substs == substs
+                                    ty::Alias(_, ty::AliasTy { def_id, substs: alias_substs, .. })
+                                    if def_id == rpit_def_id && substs == alias_substs
                                 ));
                                 ty::PredicateKind::Clause(ty::Clause::Trait(
                                     trait_pred.with_self_ty(self.tcx, ty),
@@ -548,8 +548,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             ty::PredicateKind::Clause(ty::Clause::Projection(mut proj_pred)) => {
                                 assert!(matches!(
                                     *proj_pred.projection_ty.self_ty().kind(),
-                                    ty::Alias(_, ty::AliasTy { def_id, substs, .. })
-                                    if def_id == rpit_def_id && substs == substs
+                                    ty::Alias(_, ty::AliasTy { def_id, substs: alias_substs, .. })
+                                    if def_id == rpit_def_id && substs == alias_substs
                                 ));
                                 proj_pred = proj_pred.with_self_ty(self.tcx, ty);
                                 ty::PredicateKind::Clause(ty::Clause::Projection(proj_pred))
diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs
index 6a0d5c011..5235710a2 100644
--- a/compiler/rustc_hir_typeck/src/callee.rs
+++ b/compiler/rustc_hir_typeck/src/callee.rs
@@ -311,9 +311,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let fn_decl_span = if hir.body(body).generator_kind
                 == Some(hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Closure))
             {
-                // Actually need to unwrap a few more layers of HIR to get to
+                // Actually need to unwrap one more layer of HIR to get to
                 // the _real_ closure...
-                let async_closure = hir.parent_id(hir.parent_id(parent_hir_id));
+                let async_closure = hir.parent_id(parent_hir_id);
                 if let hir::Node::Expr(hir::Expr {
                     kind: hir::ExprKind::Closure(&hir::Closure { fn_decl_span, .. }),
                     ..
@@ -668,7 +668,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         if !self.maybe_suggest_bad_array_definition(&mut err, call_expr, callee_expr) {
             if let Some((maybe_def, output_ty, _)) = self.extract_callable_info(callee_ty)
-                && !self.type_is_sized_modulo_regions(self.param_env, output_ty, callee_expr.span)
+                && !self.type_is_sized_modulo_regions(self.param_env, output_ty)
             {
                 let descr = match maybe_def {
                     DefIdOrName::DefId(def_id) => self.tcx.def_descr(def_id),
diff --git a/compiler/rustc_hir_typeck/src/cast.rs b/compiler/rustc_hir_typeck/src/cast.rs
index 316c2a7ee..1481c038c 100644
--- a/compiler/rustc_hir_typeck/src/cast.rs
+++ b/compiler/rustc_hir_typeck/src/cast.rs
@@ -96,20 +96,22 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let t = self.resolve_vars_if_possible(t);
         t.error_reported()?;
 
-        if self.type_is_sized_modulo_regions(self.param_env, t, span) {
+        if self.type_is_sized_modulo_regions(self.param_env, t) {
             return Ok(Some(PointerKind::Thin));
         }
 
         Ok(match *t.kind() {
             ty::Slice(_) | ty::Str => Some(PointerKind::Length),
             ty::Dynamic(ref tty, _, ty::Dyn) => Some(PointerKind::VTable(tty.principal_def_id())),
-            ty::Adt(def, substs) if def.is_struct() => match def.non_enum_variant().fields.last() {
-                None => Some(PointerKind::Thin),
-                Some(f) => {
-                    let field_ty = self.field_ty(span, f, substs);
-                    self.pointer_kind(field_ty, span)?
+            ty::Adt(def, substs) if def.is_struct() => {
+                match def.non_enum_variant().fields.raw.last() {
+                    None => Some(PointerKind::Thin),
+                    Some(f) => {
+                        let field_ty = self.field_ty(span, f, substs);
+                        self.pointer_kind(field_ty, span)?
+                    }
                 }
-            },
+            }
             ty::Tuple(fields) => match fields.last() {
                 None => Some(PointerKind::Thin),
                 Some(&f) => self.pointer_kind(f, span)?,
@@ -722,7 +724,7 @@ impl<'a, 'tcx> CastCheck<'tcx> {
 
         debug!("check_cast({}, {:?} as {:?})", self.expr.hir_id, self.expr_ty, self.cast_ty);
 
-        if !fcx.type_is_sized_modulo_regions(fcx.param_env, self.cast_ty, self.span)
+        if !fcx.type_is_sized_modulo_regions(fcx.param_env, self.cast_ty)
             && !self.cast_ty.has_infer_types()
         {
             self.report_cast_to_unsized_type(fcx);
diff --git a/compiler/rustc_hir_typeck/src/closure.rs b/compiler/rustc_hir_typeck/src/closure.rs
index d84fabb78..8c2495e1d 100644
--- a/compiler/rustc_hir_typeck/src/closure.rs
+++ b/compiler/rustc_hir_typeck/src/closure.rs
@@ -2,13 +2,12 @@
 
 use super::{check_fn, Expectation, FnCtxt, GeneratorTypes};
 
-use hir::def::DefKind;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir_analysis::astconv::AstConv;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use rustc_infer::infer::LateBoundRegionConversionTime;
+use rustc_infer::infer::{DefineOpaqueTypes, LateBoundRegionConversionTime};
 use rustc_infer::infer::{InferOk, InferResult};
 use rustc_macros::{TypeFoldable, TypeVisitable};
 use rustc_middle::ty::subst::InternalSubsts;
@@ -205,15 +204,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let mut expected_sig = None;
         let mut expected_kind = None;
 
-        for obligation in traits::elaborate_predicates_with_span(
+        for (pred, span) in traits::elaborate(
             self.tcx,
             // Reverse the obligations here, since `elaborate_*` uses a stack,
             // and we want to keep inference generally in the same order of
             // the registered obligations.
             predicates.rev(),
-        ) {
-            debug!(?obligation.predicate);
-            let bound_predicate = obligation.predicate.kind();
+        )
+        // We only care about self bounds
+        .filter_only_self()
+        {
+            debug!(?pred);
+            let bound_predicate = pred.kind();
 
             // Given a Projection predicate, we can potentially infer
             // the complete signature.
@@ -221,9 +223,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 && let ty::PredicateKind::Clause(ty::Clause::Projection(proj_predicate)) = bound_predicate.skip_binder()
             {
                 let inferred_sig = self.normalize(
-                    obligation.cause.span,
+                    span,
                     self.deduce_sig_from_projection(
-                    Some(obligation.cause.span),
+                    Some(span),
                         bound_predicate.rebind(proj_predicate),
                     ),
                 );
@@ -398,7 +400,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ///
     /// Here:
     /// - E would be `fn(&u32) -> &u32`.
-    /// - S would be `fn(&u32) ->
+    /// - S would be `fn(&u32) -> ?T`
     /// - E' is `&'!0 u32 -> &'!0 u32`
     /// - S' is `&'?0 u32 -> ?T`
     ///
@@ -563,10 +565,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ) {
                 // Check that E' = S'.
                 let cause = self.misc(hir_ty.span);
-                let InferOk { value: (), obligations } = self
-                    .at(&cause, self.param_env)
-                    .define_opaque_types(true)
-                    .eq(*expected_ty, supplied_ty)?;
+                let InferOk { value: (), obligations } = self.at(&cause, self.param_env).eq(
+                    DefineOpaqueTypes::Yes,
+                    *expected_ty,
+                    supplied_ty,
+                )?;
                 all_obligations.extend(obligations);
             }
 
@@ -576,10 +579,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 supplied_sig.output(),
             );
             let cause = &self.misc(decl.output.span());
-            let InferOk { value: (), obligations } = self
-                .at(cause, self.param_env)
-                .define_opaque_types(true)
-                .eq(expected_sigs.liberated_sig.output(), supplied_output_ty)?;
+            let InferOk { value: (), obligations } = self.at(cause, self.param_env).eq(
+                DefineOpaqueTypes::Yes,
+                expected_sigs.liberated_sig.output(),
+                supplied_output_ty,
+            )?;
             all_obligations.extend(obligations);
 
             let inputs = inputs.into_iter().map(|ty| self.resolve_vars_if_possible(ty));
@@ -713,14 +717,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 .subst_iter_copied(self.tcx, substs)
                 .find_map(|(p, s)| get_future_output(p, s))?,
             ty::Error(_) => return None,
-            ty::Alias(ty::Projection, proj)
-                if self.tcx.def_kind(proj.def_id) == DefKind::ImplTraitPlaceholder =>
-            {
-                self.tcx
-                    .bound_explicit_item_bounds(proj.def_id)
-                    .subst_iter_copied(self.tcx, proj.substs)
-                    .find_map(|(p, s)| get_future_output(p, s))?
-            }
+            ty::Alias(ty::Projection, proj) if self.tcx.is_impl_trait_in_trait(proj.def_id) => self
+                .tcx
+                .bound_explicit_item_bounds(proj.def_id)
+                .subst_iter_copied(self.tcx, proj.substs)
+                .find_map(|(p, s)| get_future_output(p, s))?,
             _ => span_bug!(
                 self.tcx.def_span(expr_def_id),
                 "async fn generator return type not an inference variable: {ret_ty}"
diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs
index 00b86890b..8fa3bcd68 100644
--- a/compiler/rustc_hir_typeck/src/coercion.rs
+++ b/compiler/rustc_hir_typeck/src/coercion.rs
@@ -45,8 +45,8 @@ use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::Expr;
 use rustc_hir_analysis::astconv::AstConv;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use rustc_infer::infer::{Coercion, InferOk, InferResult};
-use rustc_infer::traits::Obligation;
+use rustc_infer::infer::{Coercion, DefineOpaqueTypes, InferOk, InferResult};
+use rustc_infer::traits::{Obligation, PredicateObligation};
 use rustc_middle::lint::in_external_macro;
 use rustc_middle::ty::adjustment::{
     Adjust, Adjustment, AllowTwoPhase, AutoBorrow, AutoBorrowMutability, PointerCast,
@@ -143,11 +143,11 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
     fn unify(&self, a: Ty<'tcx>, b: Ty<'tcx>) -> InferResult<'tcx, Ty<'tcx>> {
         debug!("unify(a: {:?}, b: {:?}, use_lub: {})", a, b, self.use_lub);
         self.commit_if_ok(|_| {
-            let at = self.at(&self.cause, self.fcx.param_env).define_opaque_types(true);
+            let at = self.at(&self.cause, self.fcx.param_env);
             if self.use_lub {
-                at.lub(b, a)
+                at.lub(DefineOpaqueTypes::Yes, b, a)
             } else {
-                at.sup(b, a)
+                at.sup(DefineOpaqueTypes::Yes, b, a)
                     .map(|InferOk { value: (), obligations }| InferOk { value: a, obligations })
             }
         })
@@ -175,7 +175,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
             // so this will have the side-effect of making sure we have no ambiguities
             // due to `[type error]` and `_` not coercing together.
             let _ = self.commit_if_ok(|_| {
-                self.at(&self.cause, self.param_env).define_opaque_types(true).eq(a, b)
+                self.at(&self.cause, self.param_env).eq(DefineOpaqueTypes::Yes, a, b)
             });
             return success(vec![], self.fcx.tcx.ty_error(guar), vec![]);
         }
@@ -597,13 +597,11 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
         // and almost never more than 3. By using a SmallVec we avoid an
         // allocation, at the (very small) cost of (occasionally) having to
         // shift subsequent elements down when removing the front element.
-        let mut queue: SmallVec<[_; 4]> = smallvec![traits::predicate_for_trait_def(
+        let mut queue: SmallVec<[PredicateObligation<'tcx>; 4]> = smallvec![Obligation::new(
             self.tcx,
-            self.fcx.param_env,
             cause,
-            coerce_unsized_did,
-            0,
-            [coerce_source, coerce_target]
+            self.fcx.param_env,
+            self.tcx.mk_trait_ref(coerce_unsized_did, [coerce_source, coerce_target])
         )];
 
         let mut has_unsized_tuple_coercion = false;
@@ -651,9 +649,9 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
                         let self_ty = trait_pred.skip_binder().self_ty();
                         let unsize_ty = trait_pred.skip_binder().trait_ref.substs[1].expect_ty();
                         debug!("coerce_unsized: ambiguous unsize case for {:?}", trait_pred);
-                        match (&self_ty.kind(), &unsize_ty.kind()) {
-                            (ty::Infer(ty::TyVar(v)), ty::Dynamic(..))
-                                if self.type_var_is_sized(*v) =>
+                        match (self_ty.kind(), unsize_ty.kind()) {
+                            (&ty::Infer(ty::TyVar(v)), ty::Dynamic(..))
+                                if self.type_var_is_sized(v) =>
                             {
                                 debug!("coerce_unsized: have sized infer {:?}", v);
                                 coercion.obligations.push(obligation);
@@ -1101,9 +1099,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     (ty::FnDef(..), ty::FnDef(..)) => {
                         // Don't reify if the function types have a LUB, i.e., they
                         // are the same function and their parameters have a LUB.
-                        match self
-                            .commit_if_ok(|_| self.at(cause, self.param_env).lub(prev_ty, new_ty))
-                        {
+                        match self.commit_if_ok(|_| {
+                            self.at(cause, self.param_env).lub(
+                                DefineOpaqueTypes::No,
+                                prev_ty,
+                                new_ty,
+                            )
+                        }) {
                             // We have a LUB of prev_ty and new_ty, just return it.
                             Ok(ok) => return Ok(self.register_infer_ok_obligations(ok)),
                             Err(_) => {
@@ -1153,7 +1155,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let sig = self
                 .at(cause, self.param_env)
                 .trace(prev_ty, new_ty)
-                .lub(a_sig, b_sig)
+                .lub(DefineOpaqueTypes::No, a_sig, b_sig)
                 .map(|ok| self.register_infer_ok_obligations(ok))?;
 
             // Reify both sides and return the reified fn pointer type.
@@ -1237,7 +1239,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 );
 
                 return self
-                    .commit_if_ok(|_| self.at(cause, self.param_env).lub(prev_ty, new_ty))
+                    .commit_if_ok(|_| {
+                        self.at(cause, self.param_env).lub(DefineOpaqueTypes::No, prev_ty, new_ty)
+                    })
                     .map(|ok| self.register_infer_ok_obligations(ok));
             }
         }
@@ -1248,8 +1252,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 if let Some(e) = first_error {
                     Err(e)
                 } else {
-                    self.commit_if_ok(|_| self.at(cause, self.param_env).lub(prev_ty, new_ty))
-                        .map(|ok| self.register_infer_ok_obligations(ok))
+                    self.commit_if_ok(|_| {
+                        self.at(cause, self.param_env).lub(DefineOpaqueTypes::No, prev_ty, new_ty)
+                    })
+                    .map(|ok| self.register_infer_ok_obligations(ok))
                 }
             }
             Ok(ok) => {
@@ -1487,8 +1493,12 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
             assert!(expression_ty.is_unit(), "if let hack without unit type");
             fcx.at(cause, fcx.param_env)
                 // needed for tests/ui/type-alias-impl-trait/issue-65679-inst-opaque-ty-from-val-twice.rs
-                .define_opaque_types(true)
-                .eq_exp(label_expression_as_expected, expression_ty, self.merged_ty())
+                .eq_exp(
+                    DefineOpaqueTypes::Yes,
+                    label_expression_as_expected,
+                    expression_ty,
+                    self.merged_ty(),
+                )
                 .map(|infer_ok| {
                     fcx.register_infer_ok_obligations(infer_ok);
                     expression_ty
@@ -1710,12 +1720,13 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
                     fcx.suggest_semicolon_at_end(cond_expr.span, &mut err);
                 }
             }
-            fcx.get_node_fn_decl(parent).map(|(fn_decl, _, is_main)| (fn_decl, is_main))
+            fcx.get_node_fn_decl(parent)
+                .map(|(fn_id, fn_decl, _, is_main)| (fn_id, fn_decl, is_main))
         } else {
             fcx.get_fn_decl(parent_id)
         };
 
-        if let Some((fn_decl, can_suggest)) = fn_decl {
+        if let Some((fn_id, fn_decl, can_suggest)) = fn_decl {
             if blk_id.is_none() {
                 pointing_at_return_type |= fcx.suggest_missing_return_type(
                     &mut err,
@@ -1723,7 +1734,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
                     expected,
                     found,
                     can_suggest,
-                    fcx.tcx.hir().get_parent_item(id).into(),
+                    fn_id,
                 );
             }
             if !pointing_at_return_type {
@@ -1734,17 +1745,11 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
         let parent_id = fcx.tcx.hir().get_parent_item(id);
         let parent_item = fcx.tcx.hir().get_by_def_id(parent_id.def_id);
 
-        if let (Some(expr), Some(_), Some((fn_decl, _, _))) =
+        if let (Some(expr), Some(_), Some((fn_id, fn_decl, _, _))) =
             (expression, blk_id, fcx.get_node_fn_decl(parent_item))
         {
             fcx.suggest_missing_break_or_return_expr(
-                &mut err,
-                expr,
-                fn_decl,
-                expected,
-                found,
-                id,
-                parent_id.into(),
+                &mut err, expr, fn_decl, expected, found, id, fn_id,
             );
         }
 
@@ -1870,7 +1875,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
     }
 
     fn is_return_ty_unsized<'a>(&self, fcx: &FnCtxt<'a, 'tcx>, blk_id: hir::HirId) -> bool {
-        if let Some((fn_decl, _)) = fcx.get_fn_decl(blk_id)
+        if let Some((_, fn_decl, _)) = fcx.get_fn_decl(blk_id)
             && let hir::FnRetTy::Return(ty) = fn_decl.output
             && let ty = fcx.astconv().ast_ty_to_ty( ty)
             && let ty::Dynamic(..) = ty.kind()
diff --git a/compiler/rustc_hir_typeck/src/demand.rs b/compiler/rustc_hir_typeck/src/demand.rs
index 7ba57b3b7..13442c316 100644
--- a/compiler/rustc_hir_typeck/src/demand.rs
+++ b/compiler/rustc_hir_typeck/src/demand.rs
@@ -1,6 +1,5 @@
 use crate::FnCtxt;
 use rustc_ast::util::parser::PREC_POSTFIX;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::MultiSpan;
 use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed};
 use rustc_hir as hir;
@@ -8,19 +7,18 @@ use rustc_hir::def::CtorKind;
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::{is_range_literal, Node};
-use rustc_infer::infer::InferOk;
+use rustc_infer::infer::{DefineOpaqueTypes, InferOk};
 use rustc_middle::lint::in_external_macro;
 use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::ty::adjustment::AllowTwoPhase;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
-use rustc_middle::ty::fold::{BottomUpFolder, TypeFolder};
-use rustc_middle::ty::print::{with_forced_trimmed_paths, with_no_trimmed_paths};
-use rustc_middle::ty::relate::TypeRelation;
-use rustc_middle::ty::{self, Article, AssocItem, Ty, TypeAndMut, TypeVisitableExt};
+use rustc_middle::ty::fold::BottomUpFolder;
+use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::{self, Article, AssocItem, Ty, TypeAndMut, TypeFoldable};
 use rustc_span::symbol::{sym, Symbol};
-use rustc_span::{BytePos, Span};
+use rustc_span::{BytePos, Span, DUMMY_SP};
+use rustc_target::abi::FieldIdx;
 use rustc_trait_selection::infer::InferCtxtExt as _;
-use rustc_trait_selection::traits::error_reporting::method_chain::CollectAllMismatches;
 use rustc_trait_selection::traits::ObligationCause;
 
 use super::method::probe;
@@ -61,9 +59,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             || self.suggest_into(err, expr, expr_ty, expected)
             || self.suggest_floating_point_literal(err, expr, expected)
             || self.suggest_null_ptr_for_literal_zero_given_to_ptr_arg(err, expr, expected)
-            || self.note_result_coercion(err, expr, expected, expr_ty);
+            || self.suggest_coercing_result_via_try_operator(err, expr, expected, expr_ty);
+
         if !suggested {
-            self.point_at_expr_source_of_inferred_type(err, expr, expr_ty, expected, expr.span);
+            self.note_source_of_type_mismatch_constraint(
+                err,
+                expr,
+                TypeMismatchSource::Ty(expected),
+            );
         }
     }
 
@@ -83,7 +86,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.annotate_expected_due_to_let_ty(err, expr, error);
         self.emit_type_mismatch_suggestions(err, expr, expr_ty, expected, expected_ty_expr, error);
         self.note_type_is_not_clone(err, expected, expr_ty, expr);
-        self.note_internal_mutation_in_method(err, expr, expected, expr_ty);
+        self.note_internal_mutation_in_method(err, expr, Some(expected), expr_ty);
         self.check_for_range_as_method_call(err, expr, expr_ty, expected);
         self.check_for_binding_assigned_block_without_tail_expression(err, expr, expr_ty, expected);
         self.check_wrong_return_type_due_to_generic_arg(err, expr, expr_ty);
@@ -113,7 +116,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expected: Ty<'tcx>,
         actual: Ty<'tcx>,
     ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
-        match self.at(cause, self.param_env).define_opaque_types(true).sup(expected, actual) {
+        match self.at(cause, self.param_env).sup(DefineOpaqueTypes::Yes, expected, actual) {
             Ok(InferOk { obligations, value: () }) => {
                 self.register_predicates(obligations);
                 None
@@ -143,7 +146,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expected: Ty<'tcx>,
         actual: Ty<'tcx>,
     ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
-        match self.at(cause, self.param_env).define_opaque_types(true).eq(expected, actual) {
+        match self.at(cause, self.param_env).eq(DefineOpaqueTypes::Yes, expected, actual) {
             Ok(InferOk { obligations, value: () }) => {
                 self.register_predicates(obligations);
                 None
@@ -217,37 +220,34 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         (expected, Some(err))
     }
 
-    pub fn point_at_expr_source_of_inferred_type(
+    /// Notes the point at which a variable is constrained to some type incompatible
+    /// with some expectation given by `source`.
+    pub fn note_source_of_type_mismatch_constraint(
         &self,
         err: &mut Diagnostic,
         expr: &hir::Expr<'_>,
-        found: Ty<'tcx>,
-        expected: Ty<'tcx>,
-        mismatch_span: Span,
+        source: TypeMismatchSource<'tcx>,
     ) -> bool {
-        let map = self.tcx.hir();
+        let hir = self.tcx.hir();
 
         let hir::ExprKind::Path(hir::QPath::Resolved(None, p)) = expr.kind else { return false; };
         let [hir::PathSegment { ident, args: None, .. }] = p.segments else { return false; };
-        let hir::def::Res::Local(hir_id) = p.res else { return false; };
-        let Some(hir::Node::Pat(pat)) = map.find(hir_id) else { return false; };
-        let Some(hir::Node::Local(hir::Local {
-            ty: None,
-            init: Some(init),
-            ..
-        })) = map.find_parent(pat.hir_id) else { return false; };
-        let Some(ty) = self.node_ty_opt(init.hir_id) else { return false; };
-        if ty.is_closure() || init.span.overlaps(expr.span) || pat.span.from_expansion() {
-            return false;
-        }
+        let hir::def::Res::Local(local_hir_id) = p.res else { return false; };
+        let hir::Node::Pat(pat) = hir.get(local_hir_id) else { return false; };
+        let (init_ty_hir_id, init) = match hir.get_parent(pat.hir_id) {
+            hir::Node::Local(hir::Local { ty: Some(ty), init, .. }) => (ty.hir_id, *init),
+            hir::Node::Local(hir::Local { init: Some(init), .. }) => (init.hir_id, Some(*init)),
+            _ => return false,
+        };
+        let Some(init_ty) = self.node_ty_opt(init_ty_hir_id) else { return false; };
 
         // Locate all the usages of the relevant binding.
-        struct FindExprs<'hir> {
+        struct FindExprs<'tcx> {
             hir_id: hir::HirId,
-            uses: Vec<&'hir hir::Expr<'hir>>,
+            uses: Vec<&'tcx hir::Expr<'tcx>>,
         }
-        impl<'v> Visitor<'v> for FindExprs<'v> {
-            fn visit_expr(&mut self, ex: &'v hir::Expr<'v>) {
+        impl<'tcx> Visitor<'tcx> for FindExprs<'tcx> {
+            fn visit_expr(&mut self, ex: &'tcx hir::Expr<'tcx>) {
                 if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = ex.kind
                     && let hir::def::Res::Local(hir_id) = path.res
                     && hir_id == self.hir_id
@@ -258,180 +258,205 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         }
 
-        let mut expr_finder = FindExprs { hir_id, uses: vec![] };
-        let id = map.get_parent_item(hir_id);
-        let hir_id: hir::HirId = id.into();
-
-        let Some(node) = map.find(hir_id) else { return false; };
-        let Some(body_id) = node.body_id() else { return false; };
-        let body = map.body(body_id);
+        let mut expr_finder = FindExprs { hir_id: local_hir_id, uses: init.into_iter().collect() };
+        let body =
+            hir.body(hir.maybe_body_owned_by(self.body_id).expect("expected item to have body"));
         expr_finder.visit_expr(body.value);
-        // Hack to make equality checks on types with inference variables and regions useful.
-        let mut eraser = BottomUpFolder {
+
+        use rustc_infer::infer::type_variable::*;
+        use rustc_middle::infer::unify_key::*;
+        // Replaces all of the variables in the given type with a fresh inference variable.
+        let mut fudger = BottomUpFolder {
             tcx: self.tcx,
+            ty_op: |ty| {
+                if let ty::Infer(infer) = ty.kind() {
+                    match infer {
+                        ty::InferTy::TyVar(_) => self.next_ty_var(TypeVariableOrigin {
+                            kind: TypeVariableOriginKind::MiscVariable,
+                            span: DUMMY_SP,
+                        }),
+                        ty::InferTy::IntVar(_) => self.next_int_var(),
+                        ty::InferTy::FloatVar(_) => self.next_float_var(),
+                        _ => bug!(),
+                    }
+                } else {
+                    ty
+                }
+            },
             lt_op: |_| self.tcx.lifetimes.re_erased,
-            ct_op: |c| c,
-            ty_op: |t| match *t.kind() {
-                ty::Infer(ty::TyVar(_)) => self.tcx.mk_ty_var(ty::TyVid::from_u32(0)),
-                ty::Infer(ty::IntVar(_)) => self.tcx.mk_int_var(ty::IntVid { index: 0 }),
-                ty::Infer(ty::FloatVar(_)) => self.tcx.mk_float_var(ty::FloatVid { index: 0 }),
-                _ => t,
+            ct_op: |ct| {
+                if let ty::ConstKind::Infer(_) = ct.kind() {
+                    self.next_const_var(
+                        ct.ty(),
+                        ConstVariableOrigin {
+                            kind: ConstVariableOriginKind::MiscVariable,
+                            span: DUMMY_SP,
+                        },
+                    )
+                } else {
+                    ct
+                }
             },
         };
-        let mut prev = eraser.fold_ty(ty);
-        let mut prev_span: Option<Span> = None;
-
-        for binding in expr_finder.uses {
-            // In every expression where the binding is referenced, we will look at that
-            // expression's type and see if it is where the incorrect found type was fully
-            // "materialized" and point at it. We will also try to provide a suggestion there.
-            if let Some(hir::Node::Expr(expr)
-            | hir::Node::Stmt(hir::Stmt {
-                kind: hir::StmtKind::Expr(expr) | hir::StmtKind::Semi(expr),
-                ..
-            })) = &map.find_parent(binding.hir_id)
-                && let hir::ExprKind::MethodCall(segment, rcvr, args, _span) = expr.kind
-                && rcvr.hir_id == binding.hir_id
-                && let Some(def_id) = self.typeck_results.borrow().type_dependent_def_id(expr.hir_id)
-            {
-                // We special case methods, because they can influence inference through the
-                // call's arguments and we can provide a more explicit span.
-                let sig = self.tcx.fn_sig(def_id).subst_identity();
-                let def_self_ty = sig.input(0).skip_binder();
-                let param_tys = sig.inputs().skip_binder().iter().skip(1);
-                // If there's an arity mismatch, pointing out the call as the source of an inference
-                // can be misleading, so we skip it.
-                if param_tys.len() != args.len() {
-                    continue;
-                }
-                let rcvr_ty = self.node_ty(rcvr.hir_id);
-                // Get the evaluated type *after* calling the method call, so that the influence
-                // of the arguments can be reflected in the receiver type. The receiver
-                // expression has the type *before* theis analysis is done.
-                let ty = match self.lookup_probe_for_diagnostic(
-                    segment.ident,
-                    rcvr_ty,
-                    expr,
-                    probe::ProbeScope::TraitsInScope,
-                    None,
-                ) {
-                    Ok(pick) => eraser.fold_ty(pick.self_ty),
-                    Err(_) => rcvr_ty,
+
+        let expected_ty = match source {
+            TypeMismatchSource::Ty(expected_ty) => expected_ty,
+            // Try to deduce what the possible value of `expr` would be if the
+            // incompatible arg were compatible. For example, given `Vec<i32>`
+            // and `vec.push(1u32)`, we ideally want to deduce that the type of
+            // `vec` *should* have been `Vec<u32>`. This will allow us to then
+            // run the subsequent code with this expectation, finding out exactly
+            // when this type diverged from our expectation.
+            TypeMismatchSource::Arg { call_expr, incompatible_arg: idx } => {
+                let hir::ExprKind::MethodCall(segment, _, args, _) = call_expr.kind else {
+                    return false;
                 };
-                // Remove one layer of references to account for `&mut self` and
-                // `&self`, so that we can compare it against the binding.
-                let (ty, def_self_ty) = match (ty.kind(), def_self_ty.kind()) {
-                    (ty::Ref(_, ty, a), ty::Ref(_, self_ty, b)) if a == b => (*ty, *self_ty),
-                    _ => (ty, def_self_ty),
+                let Some(arg_ty) = self.node_ty_opt(args[idx].hir_id) else {
+                    return false;
                 };
-                let mut param_args = FxHashMap::default();
-                let mut param_expected = FxHashMap::default();
-                let mut param_found = FxHashMap::default();
-                if self.can_eq(self.param_env, ty, found) {
-                    // We only point at the first place where the found type was inferred.
-                    for (param_ty, arg) in param_tys.zip(args) {
-                        if def_self_ty.contains(*param_ty) && let ty::Param(_) = param_ty.kind() {
-                            // We found an argument that references a type parameter in `Self`,
-                            // so we assume that this is the argument that caused the found
-                            // type, which we know already because of `can_eq` above was first
-                            // inferred in this method call.
-                            let arg_ty = self.node_ty(arg.hir_id);
-                            if !arg.span.overlaps(mismatch_span) {
-                                err.span_label(
-                                    arg.span,
-                                    &format!(
-                                        "this is of type `{arg_ty}`, which causes `{ident}` to be \
-                                        inferred as `{ty}`",
-                                    ),
-                                );
-                            }
-                            param_args.insert(param_ty, (arg, arg_ty));
-                        }
-                    }
+                let possible_rcvr_ty = expr_finder.uses.iter().find_map(|binding| {
+                    let possible_rcvr_ty = self.node_ty_opt(binding.hir_id)?;
+                    // Fudge the receiver, so we can do new inference on it.
+                    let possible_rcvr_ty = possible_rcvr_ty.fold_with(&mut fudger);
+                    let method = self
+                        .lookup_method(
+                            possible_rcvr_ty,
+                            segment,
+                            DUMMY_SP,
+                            call_expr,
+                            binding,
+                            args,
+                        )
+                        .ok()?;
+                    // Unify the method signature with our incompatible arg, to
+                    // do inference in the *opposite* direction and to find out
+                    // what our ideal rcvr ty would look like.
+                    let _ = self
+                        .at(&ObligationCause::dummy(), self.param_env)
+                        .eq(DefineOpaqueTypes::No, method.sig.inputs()[idx + 1], arg_ty)
+                        .ok()?;
+                    self.select_obligations_where_possible(|errs| {
+                        // Yeet the errors, we're already reporting errors.
+                        errs.clear();
+                    });
+                    Some(self.resolve_vars_if_possible(possible_rcvr_ty))
+                });
+                if let Some(rcvr_ty) = possible_rcvr_ty {
+                    rcvr_ty
+                } else {
+                    return false;
                 }
+            }
+        };
 
-                // Here we find, for a type param `T`, the type that `T` is in the current
-                // method call *and* in the original expected type. That way, we can see if we
-                // can give any structured suggestion for the function argument.
-                let mut c = CollectAllMismatches {
-                    infcx: &self.infcx,
-                    param_env: self.param_env,
-                    errors: vec![],
+        // If our expected_ty does not equal init_ty, then it *began* as incompatible.
+        // No need to note in this case...
+        if !self.can_eq(self.param_env, expected_ty, init_ty.fold_with(&mut fudger)) {
+            return false;
+        }
+
+        for window in expr_finder.uses.windows(2) {
+            // Bindings always update their recorded type after the fact, so we
+            // need to look at the *following* usage's type to see when the
+            // binding became incompatible.
+            let [binding, next_usage] = *window else { continue; };
+
+            // Don't go past the binding (always gonna be a nonsense label if so)
+            if binding.hir_id == expr.hir_id {
+                break;
+            }
+
+            let Some(next_use_ty) = self.node_ty_opt(next_usage.hir_id) else { continue; };
+
+            // If the type is not constrained in a way making it not possible to
+            // equate with `expected_ty` by this point, skip.
+            if self.can_eq(self.param_env, expected_ty, next_use_ty.fold_with(&mut fudger)) {
+                continue;
+            }
+
+            if let hir::Node::Expr(parent_expr) = hir.get_parent(binding.hir_id)
+                && let hir::ExprKind::MethodCall(segment, rcvr, args, _) = parent_expr.kind
+                && rcvr.hir_id == binding.hir_id
+            {
+                // If our binding became incompatible while it was a receiver
+                // to a method call, we may be able to make a better guess to
+                // the source of a type mismatch.
+                let Some(rcvr_ty) = self.node_ty_opt(rcvr.hir_id) else { continue; };
+                let rcvr_ty = rcvr_ty.fold_with(&mut fudger);
+                let Ok(method) =
+                    self.lookup_method(rcvr_ty, segment, DUMMY_SP, parent_expr, rcvr, args)
+                else {
+                    continue;
                 };
-                let _ = c.relate(def_self_ty, ty);
-                for error in c.errors {
-                    if let TypeError::Sorts(error) = error {
-                        param_found.insert(error.expected, error.found);
-                    }
-                }
-                c.errors = vec![];
-                let _ = c.relate(def_self_ty, expected);
-                for error in c.errors {
-                    if let TypeError::Sorts(error) = error {
-                        param_expected.insert(error.expected, error.found);
-                    }
-                }
-                for (param, (arg, arg_ty)) in param_args.iter() {
-                    let Some(expected) = param_expected.get(param) else { continue; };
-                    let Some(found) = param_found.get(param) else { continue; };
-                    if !self.can_eq(self.param_env, *arg_ty, *found) { continue; }
-                    self.emit_coerce_suggestions(err, arg, *found, *expected, None, None);
-                }
 
-                let ty = eraser.fold_ty(ty);
-                if ty.references_error() {
-                    break;
-                }
-                if ty != prev
-                    && param_args.is_empty()
-                    && self.can_eq(self.param_env, ty, found)
+                let ideal_rcvr_ty = rcvr_ty.fold_with(&mut fudger);
+                let ideal_method = self
+                    .lookup_method(ideal_rcvr_ty, segment, DUMMY_SP, parent_expr, rcvr, args)
+                    .ok()
+                    .and_then(|method| {
+                        let _ = self.at(&ObligationCause::dummy(), self.param_env)
+                            .eq(DefineOpaqueTypes::No, ideal_rcvr_ty, expected_ty)
+                            .ok()?;
+                        Some(method)
+                    });
+
+                // Find what argument caused our rcvr to become incompatible
+                // with the expected ty.
+                for (idx, (expected_arg_ty, arg_expr)) in
+                    std::iter::zip(&method.sig.inputs()[1..], args).enumerate()
                 {
-                    // We only point at the first place where the found type was inferred.
-                    if !segment.ident.span.overlaps(mismatch_span) {
+                    let Some(arg_ty) = self.node_ty_opt(arg_expr.hir_id) else { continue; };
+                    let arg_ty = arg_ty.fold_with(&mut fudger);
+                    let _ = self.try_coerce(
+                        arg_expr,
+                        arg_ty,
+                        *expected_arg_ty,
+                        AllowTwoPhase::No,
+                        None,
+                    );
+                    self.select_obligations_where_possible(|errs| {
+                        // Yeet the errors, we're already reporting errors.
+                        errs.clear();
+                    });
+                    // If our rcvr, after inference due to unifying the signature
+                    // with the expected argument type, is still compatible with
+                    // the rcvr, then it must've not been the source of blame.
+                    if self.can_eq(self.param_env, rcvr_ty, expected_ty) {
+                        continue;
+                    }
+                    err.span_label(arg_expr.span, format!("this argument has type `{arg_ty}`..."));
                     err.span_label(
-                        segment.ident.span,
-                        with_forced_trimmed_paths!(format!(
-                            "here the type of `{ident}` is inferred to be `{ty}`",
-                        )),
-                    );}
-                    break;
-                } else if !param_args.is_empty() {
-                    break;
-                }
-                prev = ty;
-            } else {
-                let ty = eraser.fold_ty(self.node_ty(binding.hir_id));
-                if ty.references_error() {
-                    break;
-                }
-                if ty != prev
-                    && let Some(span) = prev_span
-                    && self.can_eq(self.param_env, ty, found)
-                {
-                    // We only point at the first place where the found type was inferred.
-                    // We use the *previous* span because if the type is known *here* it means
-                    // it was *evaluated earlier*. We don't do this for method calls because we
-                    // evaluate the method's self type eagerly, but not in any other case.
-                    if !span.overlaps(mismatch_span) {
-                        err.span_label(
-                            span,
-                            with_forced_trimmed_paths!(format!(
-                                "here the type of `{ident}` is inferred to be `{ty}`",
-                            )),
+                        binding.span,
+                        format!("... which causes `{ident}` to have type `{next_use_ty}`"),
+                    );
+                    // Using our "ideal" method signature, suggest a fix to this
+                    // blame arg, if possible. Don't do this if we're coming from
+                    // arg mismatch code, because we'll possibly suggest a mutually
+                    // incompatible fix at the original mismatch site.
+                    if matches!(source, TypeMismatchSource::Ty(_))
+                        && let Some(ideal_method) = ideal_method
+                    {
+                        self.emit_type_mismatch_suggestions(
+                            err,
+                            arg_expr,
+                            arg_ty,
+                            self.resolve_vars_if_possible(ideal_method.sig.inputs()[idx + 1]),
+                            None,
+                            None,
                         );
                     }
-                    break;
+                    return true;
                 }
-                prev = ty;
-            }
-            if binding.hir_id == expr.hir_id {
-                // Do not look at expressions that come after the expression we were originally
-                // evaluating and had a type error.
-                break;
             }
-            prev_span = Some(binding.span);
+            err.span_label(
+                binding.span,
+                format!("here the type of `{ident}` is inferred to be `{next_use_ty}`"),
+            );
+            return true;
         }
-        true
+
+        // We must've not found something that constrained the expr.
+        false
     }
 
     fn annotate_expected_due_to_let_ty(
@@ -707,7 +732,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         );
     }
 
-    pub(crate) fn note_result_coercion(
+    pub(crate) fn suggest_coercing_result_via_try_operator(
         &self,
         err: &mut Diagnostic,
         expr: &hir::Expr<'tcx>,
@@ -850,7 +875,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     variant.fields.len() == 1
                 })
                 .filter_map(|variant| {
-                    let sole_field = &variant.fields[0];
+                    let sole_field = &variant.fields[FieldIdx::from_u32(0)];
 
                     let field_is_local = sole_field.did.is_local();
                     let field_is_accessible =
@@ -1480,14 +1505,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                     // For this suggestion to make sense, the type would need to be `Copy`,
                     // or we have to be moving out of a `Box<T>`
-                    if self.type_is_copy_modulo_regions(self.param_env, expected, sp)
+                    if self.type_is_copy_modulo_regions(self.param_env, expected)
                         // FIXME(compiler-errors): We can actually do this if the checked_ty is
                         // `steps` layers of boxes, not just one, but this is easier and most likely.
                         || (checked_ty.is_box() && steps == 1)
                     {
                         let deref_kind = if checked_ty.is_box() {
                             "unboxing the value"
-                        } else if checked_ty.is_region_ptr() {
+                        } else if checked_ty.is_ref() {
                             "dereferencing the borrow"
                         } else {
                             "dereferencing the type"
@@ -2093,3 +2118,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 }
+
+pub enum TypeMismatchSource<'tcx> {
+    /// Expected the binding to have the given type, but it was found to have
+    /// a different type. Find out when that type first became incompatible.
+    Ty(Ty<'tcx>),
+    /// When we fail during method argument checking, try to find out if a previous
+    /// expression has constrained the method's receiver in a way that makes the
+    /// argument's type incompatible.
+    Arg { call_expr: &'tcx hir::Expr<'tcx>, incompatible_arg: usize },
+}
diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index 7fc4ccb04..6ffa0134f 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -36,6 +36,7 @@ use rustc_hir_analysis::astconv::AstConv as _;
 use rustc_hir_analysis::check::ty_kind_suggestion;
 use rustc_infer::infer;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
+use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::infer::InferOk;
 use rustc_infer::traits::ObligationCause;
 use rustc_middle::middle::stability;
@@ -49,6 +50,7 @@ use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::hygiene::DesugaringKind;
 use rustc_span::source_map::{Span, Spanned};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
+use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi::RustIntrinsic;
 use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::traits::{self, ObligationCauseCode};
@@ -118,7 +120,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         ty
     }
 
-    pub(super) fn check_expr_coercable_to_type(
+    pub(super) fn check_expr_coercible_to_type(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
         expected: Ty<'tcx>,
@@ -230,7 +232,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let ty = ensure_sufficient_stack(|| match &expr.kind {
             hir::ExprKind::Path(
-                qpath @ hir::QPath::Resolved(..) | qpath @ hir::QPath::TypeRelative(..),
+                qpath @ (hir::QPath::Resolved(..) | hir::QPath::TypeRelative(..)),
             ) => self.check_expr_path(qpath, expr, args),
             _ => self.check_expr_kind(expr, expected),
         });
@@ -283,7 +285,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let tcx = self.tcx;
         match expr.kind {
-            ExprKind::Box(subexpr) => self.check_expr_box(subexpr, expected),
             ExprKind::Lit(ref lit) => self.check_lit(&lit, expected),
             ExprKind::Binary(op, lhs, rhs) => self.check_binop(expr, op, lhs, rhs, expected),
             ExprKind::Assign(lhs, rhs, span) => {
@@ -358,16 +359,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    fn check_expr_box(&self, expr: &'tcx hir::Expr<'tcx>, expected: Expectation<'tcx>) -> Ty<'tcx> {
-        let expected_inner = expected.to_option(self).map_or(NoExpectation, |ty| match ty.kind() {
-            ty::Adt(def, _) if def.is_box() => Expectation::rvalue_hint(self, ty.boxed_ty()),
-            _ => NoExpectation,
-        });
-        let referent_ty = self.check_expr_with_expectation(expr, expected_inner);
-        self.require_type_is_sized(referent_ty, expr.span, traits::SizedBoxType);
-        self.tcx.mk_box(referent_ty)
-    }
-
     fn check_expr_unary(
         &self,
         unop: hir::UnOp,
@@ -798,7 +789,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.ret_coercion_span.set(Some(expr.span));
             }
             let cause = self.cause(expr.span, ObligationCauseCode::ReturnNoExpression);
-            if let Some((fn_decl, _)) = self.get_fn_decl(expr.hir_id) {
+            if let Some((_, fn_decl, _)) = self.get_fn_decl(expr.hir_id) {
                 coercion.coerce_forced_unit(
                     self,
                     &cause,
@@ -1137,7 +1128,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         };
 
-        // This is (basically) inlined `check_expr_coercable_to_type`, but we want
+        // This is (basically) inlined `check_expr_coercible_to_type`, but we want
         // to suggest an additional fixup here in `suggest_deref_binop`.
         let rhs_ty = self.check_expr_with_hint(&rhs, lhs_ty);
         if let (_, Some(mut diag)) =
@@ -1410,7 +1401,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let (element_ty, t) = match uty {
             Some(uty) => {
-                self.check_expr_coercable_to_type(&element, uty, None);
+                self.check_expr_coercible_to_type(&element, uty, None);
                 (uty, uty)
             }
             None => {
@@ -1487,7 +1478,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let elt_ts_iter = elts.iter().enumerate().map(|(i, e)| match flds {
             Some(fs) if i < fs.len() => {
                 let ety = fs[i];
-                self.check_expr_coercable_to_type(&e, ety, None);
+                self.check_expr_coercible_to_type(&e, ety, None);
                 ety
             }
             _ => self.check_expr_with_expectation(&e, NoExpectation),
@@ -1571,8 +1562,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let mut remaining_fields = variant
             .fields
-            .iter()
-            .enumerate()
+            .iter_enumerated()
             .map(|(i, field)| (field.ident(tcx).normalize_to_macros_2_0(), (i, field)))
             .collect::<FxHashMap<_, _>>();
 
@@ -1683,7 +1673,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             if let Some(_) = remaining_fields.remove(&ident) {
                                 let target_ty = self.field_ty(base_expr.span, f, substs);
                                 let cause = self.misc(base_expr.span);
-                                match self.at(&cause, self.param_env).sup(target_ty, fru_ty) {
+                                match self.at(&cause, self.param_env).sup(
+                                    DefineOpaqueTypes::No,
+                                    target_ty,
+                                    fru_ty,
+                                ) {
                                     Ok(InferOk { obligations, value: () }) => {
                                         self.register_predicates(obligations)
                                     }
@@ -1821,7 +1815,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         adt_ty: Ty<'tcx>,
         span: Span,
-        remaining_fields: FxHashMap<Ident, (usize, &ty::FieldDef)>,
+        remaining_fields: FxHashMap<Ident, (FieldIdx, &ty::FieldDef)>,
         variant: &'tcx ty::VariantDef,
         ast_fields: &'tcx [hir::ExprField<'tcx>],
         substs: SubstsRef<'tcx>,
@@ -2215,11 +2209,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let (ident, def_scope) =
                         self.tcx.adjust_ident_and_get_scope(field, base_def.did(), body_hir_id);
                     let fields = &base_def.non_enum_variant().fields;
-                    if let Some(index) = fields
-                        .iter()
-                        .position(|f| f.ident(self.tcx).normalize_to_macros_2_0() == ident)
+                    if let Some((index, field)) = fields
+                        .iter_enumerated()
+                        .find(|(_, f)| f.ident(self.tcx).normalize_to_macros_2_0() == ident)
                     {
-                        let field = &fields[index];
                         let field_ty = self.field_ty(expr.span, field, substs);
                         // Save the index of all fields regardless of their visibility in case
                         // of error recovery.
@@ -2236,15 +2229,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     }
                 }
                 ty::Tuple(tys) => {
-                    let fstr = field.as_str();
-                    if let Ok(index) = fstr.parse::<usize>() {
-                        if fstr == index.to_string() {
+                    if let Ok(index) = field.as_str().parse::<usize>() {
+                        if field.name == sym::integer(index) {
                             if let Some(&field_ty) = tys.get(index) {
                                 let adjustments = self.adjust_steps(&autoderef);
                                 self.apply_adjustments(base, adjustments);
                                 self.register_predicates(autoderef.into_obligations());
 
-                                self.write_field_index(expr.hir_id, index);
+                                self.write_field_index(expr.hir_id, FieldIdx::from_usize(index));
                                 return field_ty;
                             }
                         }
@@ -2818,23 +2810,26 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         "cannot index into a value of type `{base_t}`",
                     );
                     // Try to give some advice about indexing tuples.
-                    if let ty::Tuple(..) = base_t.kind() {
+                    if let ty::Tuple(types) = base_t.kind() {
                         let mut needs_note = true;
                         // If the index is an integer, we can show the actual
                         // fixed expression:
-                        if let ExprKind::Lit(ref lit) = idx.kind {
-                            if let ast::LitKind::Int(i, ast::LitIntType::Unsuffixed) = lit.node {
-                                let snip = self.tcx.sess.source_map().span_to_snippet(base.span);
-                                if let Ok(snip) = snip {
-                                    err.span_suggestion(
-                                        expr.span,
-                                        "to access tuple elements, use",
-                                        format!("{snip}.{i}"),
-                                        Applicability::MachineApplicable,
-                                    );
-                                    needs_note = false;
-                                }
+                        if let ExprKind::Lit(ref lit) = idx.kind
+                            && let ast::LitKind::Int(i, ast::LitIntType::Unsuffixed) = lit.node
+                            && i < types.len().try_into().expect("expected tuple index to be < usize length")
+                        {
+                            let snip = self.tcx.sess.source_map().span_to_snippet(base.span);
+                            if let Ok(snip) = snip {
+                                err.span_suggestion(
+                                    expr.span,
+                                    "to access tuple elements, use",
+                                    format!("{snip}.{i}"),
+                                    Applicability::MachineApplicable,
+                                );
+                                needs_note = false;
                             }
+                        } else if let ExprKind::Path(..) = idx.peel_borrows().kind {
+                            err.span_label(idx.span, "cannot access tuple elements at a variable index");
                         }
                         if needs_note {
                             err.help(
@@ -2874,7 +2869,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) -> Ty<'tcx> {
         match self.resume_yield_tys {
             Some((resume_ty, yield_ty)) => {
-                self.check_expr_coercable_to_type(&value, yield_ty, None);
+                self.check_expr_coercible_to_type(&value, yield_ty, None);
 
                 resume_ty
             }
@@ -2883,7 +2878,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // information. Hence, we check the source of the yield expression here and check its
             // value's type against `()` (this check should always hold).
             None if src.is_await() => {
-                self.check_expr_coercable_to_type(&value, self.tcx.mk_unit(), None);
+                self.check_expr_coercible_to_type(&value, self.tcx.mk_unit(), None);
                 self.tcx.mk_unit()
             }
             _ => {
diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
index b9a058d6b..ee1c6fbfd 100644
--- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
+++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
@@ -14,12 +14,11 @@ use rustc_hir as hir;
 use rustc_hir::def::Res;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::PatKind;
-use rustc_index::vec::Idx;
 use rustc_infer::infer::InferCtxt;
 use rustc_middle::hir::place::ProjectionKind;
 use rustc_middle::mir::FakeReadCause;
 use rustc_middle::ty::{self, adjustment, AdtKind, Ty, TyCtxt};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::FIRST_VARIANT;
 use ty::BorrowKind::ImmBorrow;
 
 use crate::mem_categorization as mc;
@@ -356,10 +355,6 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                 self.walk_captures(closure);
             }
 
-            hir::ExprKind::Box(ref base) => {
-                self.consume_expr(base);
-            }
-
             hir::ExprKind::Yield(value, _) => {
                 self.consume_expr(value);
             }
@@ -544,7 +539,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
         match with_place.place.ty().kind() {
             ty::Adt(adt, substs) if adt.is_struct() => {
                 // Consume those fields of the with expression that are needed.
-                for (f_index, with_field) in adt.non_enum_variant().fields.iter().enumerate() {
+                for (f_index, with_field) in adt.non_enum_variant().fields.iter_enumerated() {
                     let is_mentioned = fields
                         .iter()
                         .any(|f| self.mc.typeck_results.opt_field_index(f.hir_id) == Some(f_index));
@@ -553,7 +548,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                             &*with_expr,
                             with_place.clone(),
                             with_field.ty(self.tcx(), substs),
-                            ProjectionKind::Field(f_index as u32, VariantIdx::new(0)),
+                            ProjectionKind::Field(f_index, FIRST_VARIANT),
                         );
                         self.delegate_consume(&field_place, field_place.hir_id);
                     }
@@ -564,7 +559,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                 // struct; however, when EUV is run during typeck, it
                 // may not. This will generate an error earlier in typeck,
                 // so we can just ignore it.
-                if !self.tcx().sess.has_errors().is_some() {
+                if self.tcx().sess.has_errors().is_none() {
                     span_bug!(with_expr.span, "with expression doesn't evaluate to a struct");
                 }
             }
@@ -871,10 +866,7 @@ fn copy_or_move<'a, 'tcx>(
     mc: &mc::MemCategorizationContext<'a, 'tcx>,
     place_with_id: &PlaceWithHirId<'tcx>,
 ) -> ConsumeMode {
-    if !mc.type_is_copy_modulo_regions(
-        place_with_id.place.ty(),
-        mc.tcx().hir().span(place_with_id.hir_id),
-    ) {
+    if !mc.type_is_copy_modulo_regions(place_with_id.place.ty()) {
         ConsumeMode::Move
     } else {
         ConsumeMode::Copy
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
index 60e55c7b0..f736f7a96 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
@@ -19,13 +19,13 @@ use rustc_hir_analysis::astconv::{
 };
 use rustc_infer::infer::canonical::{Canonical, OriginalQueryValues, QueryResponse};
 use rustc_infer::infer::error_reporting::TypeAnnotationNeeded::E0282;
-use rustc_infer::infer::InferResult;
+use rustc_infer::infer::{DefineOpaqueTypes, InferResult};
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, AutoBorrow, AutoBorrowMutability};
 use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::{
-    self, AdtKind, CanonicalUserType, DefIdTree, GenericParamDefKind, Ty, TyCtxt, UserType,
+    self, AdtKind, CanonicalUserType, GenericParamDefKind, Ty, TyCtxt, UserType,
 };
 use rustc_middle::ty::{GenericArgKind, SubstsRef, UserSelfTy, UserSubsts};
 use rustc_session::lint;
@@ -33,6 +33,7 @@ use rustc_span::def_id::LocalDefId;
 use rustc_span::hygiene::DesugaringKind;
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Span;
+use rustc_target::abi::FieldIdx;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
 use rustc_trait_selection::traits::{self, NormalizeExt, ObligationCauseCode, ObligationCtxt};
 
@@ -147,7 +148,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    pub fn write_field_index(&self, hir_id: hir::HirId, index: usize) {
+    pub fn write_field_index(&self, hir_id: hir::HirId, index: FieldIdx) {
         self.typeck_results.borrow_mut().field_indices_mut().insert(hir_id, index);
     }
 
@@ -301,16 +302,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         span: Span,
         def_id: DefId,
         substs: SubstsRef<'tcx>,
-    ) -> (ty::InstantiatedPredicates<'tcx>, Vec<Span>) {
+    ) -> ty::InstantiatedPredicates<'tcx> {
         let bounds = self.tcx.predicates_of(def_id);
-        let spans: Vec<Span> = bounds.predicates.iter().map(|(_, span)| *span).collect();
         let result = bounds.instantiate(self.tcx, substs);
         let result = self.normalize(span, result);
-        debug!(
-            "instantiate_bounds(bounds={:?}, substs={:?}) = {:?}, {:?}",
-            bounds, substs, result, spans,
-        );
-        (result, spans)
+        debug!("instantiate_bounds(bounds={:?}, substs={:?}) = {:?}", bounds, substs, result);
+        result
     }
 
     pub(in super::super) fn normalize<T>(&self, span: Span, value: T) -> T
@@ -562,7 +559,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let span = self.tcx.hir().body(body_id).value.span;
             let ok = self
                 .at(&self.misc(span), self.param_env)
-                .eq(interior, witness)
+                .eq(DefineOpaqueTypes::No, interior, witness)
                 .expect("Failed to unify generator interior type");
             let mut obligations = ok.obligations;
 
@@ -581,11 +578,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     #[instrument(skip(self), level = "debug")]
     pub(in super::super) fn report_ambiguity_errors(&self) {
-        let mut errors = self.fulfillment_cx.borrow_mut().collect_remaining_errors();
+        let mut errors = self.fulfillment_cx.borrow_mut().collect_remaining_errors(self);
 
         if !errors.is_empty() {
             self.adjust_fulfillment_errors_for_expr_obligation(&mut errors);
-            self.err_ctxt().report_fulfillment_errors(&errors, self.inh.body_id);
+            self.err_ctxt().report_fulfillment_errors(&errors);
         }
     }
 
@@ -598,7 +595,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         if !result.is_empty() {
             mutate_fulfillment_errors(&mut result);
             self.adjust_fulfillment_errors_for_expr_obligation(&mut result);
-            self.err_ctxt().report_fulfillment_errors(&result, self.inh.body_id);
+            self.err_ctxt().report_fulfillment_errors(&result);
         }
     }
 
@@ -670,7 +667,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(..))
                 | ty::PredicateKind::WellFormed(..)
                 | ty::PredicateKind::ObjectSafe(..)
-                | ty::PredicateKind::AliasEq(..)
+                | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::ConstEvaluatable(..)
                 | ty::PredicateKind::ConstEquate(..)
                 // N.B., this predicate is created by breaking down a
@@ -902,56 +899,74 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         )
     }
 
-    /// Given a function `Node`, return its `FnDecl` if it exists, or `None` otherwise.
+    /// Given a function `Node`, return its  `HirId` and `FnDecl` if it exists. Given a closure
+    /// that is the child of a function, return that function's `HirId` and `FnDecl` instead.
+    /// This may seem confusing at first, but this is used in diagnostics for `async fn`,
+    /// for example, where most of the type checking actually happens within a nested closure,
+    /// but we often want access to the parent function's signature.
+    ///
+    /// Otherwise, return false.
     pub(in super::super) fn get_node_fn_decl(
         &self,
         node: Node<'tcx>,
-    ) -> Option<(&'tcx hir::FnDecl<'tcx>, Ident, bool)> {
+    ) -> Option<(hir::HirId, &'tcx hir::FnDecl<'tcx>, Ident, bool)> {
         match node {
-            Node::Item(&hir::Item { ident, kind: hir::ItemKind::Fn(ref sig, ..), .. }) => {
+            Node::Item(&hir::Item {
+                ident,
+                kind: hir::ItemKind::Fn(ref sig, ..),
+                owner_id,
+                ..
+            }) => {
                 // This is less than ideal, it will not suggest a return type span on any
                 // method called `main`, regardless of whether it is actually the entry point,
                 // but it will still present it as the reason for the expected type.
-                Some((&sig.decl, ident, ident.name != sym::main))
+                Some((
+                    hir::HirId::make_owner(owner_id.def_id),
+                    &sig.decl,
+                    ident,
+                    ident.name != sym::main,
+                ))
             }
             Node::TraitItem(&hir::TraitItem {
                 ident,
                 kind: hir::TraitItemKind::Fn(ref sig, ..),
+                owner_id,
                 ..
-            }) => Some((&sig.decl, ident, true)),
+            }) => Some((hir::HirId::make_owner(owner_id.def_id), &sig.decl, ident, true)),
             Node::ImplItem(&hir::ImplItem {
                 ident,
                 kind: hir::ImplItemKind::Fn(ref sig, ..),
+                owner_id,
                 ..
-            }) => Some((&sig.decl, ident, false)),
-            Node::Expr(&hir::Expr {
-                hir_id,
-                kind: hir::ExprKind::Closure(..),
-                ..
-            }) if let Some(Node::Expr(&hir::Expr {
-                hir_id,
-                kind: hir::ExprKind::Call(..),
-                ..
-            })) = self.tcx.hir().find_parent(hir_id) &&
-            let Some(Node::Item(&hir::Item {
+            }) => Some((hir::HirId::make_owner(owner_id.def_id), &sig.decl, ident, false)),
+            Node::Expr(&hir::Expr { hir_id, kind: hir::ExprKind::Closure(..), .. })
+                if let Some(Node::Item(&hir::Item {
+                    ident,
+                    kind: hir::ItemKind::Fn(ref sig, ..),
+                    owner_id,
+                    ..
+                })) = self.tcx.hir().find_parent(hir_id) => Some((
+                hir::HirId::make_owner(owner_id.def_id),
+                &sig.decl,
                 ident,
-                kind: hir::ItemKind::Fn(ref sig, ..),
-                ..
-            })) = self.tcx.hir().find_parent(hir_id) => {
-                Some((&sig.decl, ident, ident.name != sym::main))
-            },
+                ident.name != sym::main,
+            )),
             _ => None,
         }
     }
 
-    /// Given a `HirId`, return the `FnDecl` of the method it is enclosed by and whether a
+    /// Given a `HirId`, return the `HirId` of the enclosing function, its `FnDecl`, and whether a
     /// suggestion can be made, `None` otherwise.
-    pub fn get_fn_decl(&self, blk_id: hir::HirId) -> Option<(&'tcx hir::FnDecl<'tcx>, bool)> {
+    pub fn get_fn_decl(
+        &self,
+        blk_id: hir::HirId,
+    ) -> Option<(hir::HirId, &'tcx hir::FnDecl<'tcx>, bool)> {
         // Get enclosing Fn, if it is a function or a trait method, unless there's a `loop` or
         // `while` before reaching it, as block tail returns are not available in them.
         self.tcx.hir().get_return_block(blk_id).and_then(|blk_id| {
             let parent = self.tcx.hir().get(blk_id);
-            self.get_node_fn_decl(parent).map(|(fn_decl, _, is_main)| (fn_decl, is_main))
+            self.get_node_fn_decl(parent)
+                .map(|(fn_id, fn_decl, _, is_main)| (fn_id, fn_decl, is_main))
         })
     }
 
@@ -959,44 +974,75 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         err: &mut Diagnostic,
         expr: &hir::Expr<'_>,
-        expected: Ty<'tcx>,
+        expected: Option<Ty<'tcx>>,
         found: Ty<'tcx>,
     ) {
         if found != self.tcx.types.unit {
             return;
         }
-        if let ExprKind::MethodCall(path_segment, rcvr, ..) = expr.kind {
-            if self
-                .typeck_results
+
+        let ExprKind::MethodCall(path_segment, rcvr, ..) = expr.kind else {
+            return;
+        };
+
+        let rcvr_has_the_expected_type = self
+            .typeck_results
+            .borrow()
+            .expr_ty_adjusted_opt(rcvr)
+            .and_then(|ty| expected.map(|expected_ty| expected_ty.peel_refs() == ty.peel_refs()))
+            .unwrap_or(false);
+
+        let prev_call_mutates_and_returns_unit = || {
+            self.typeck_results
                 .borrow()
-                .expr_ty_adjusted_opt(rcvr)
-                .map_or(true, |ty| expected.peel_refs() != ty.peel_refs())
-            {
-                return;
-            }
-            let mut sp = MultiSpan::from_span(path_segment.ident.span);
-            sp.push_span_label(
-                path_segment.ident.span,
-                format!(
-                    "this call modifies {} in-place",
-                    match rcvr.kind {
-                        ExprKind::Path(QPath::Resolved(
-                            None,
-                            hir::Path { segments: [segment], .. },
-                        )) => format!("`{}`", segment.ident),
-                        _ => "its receiver".to_string(),
-                    }
-                ),
-            );
+                .type_dependent_def_id(expr.hir_id)
+                .map(|def_id| self.tcx.fn_sig(def_id).skip_binder().skip_binder())
+                .and_then(|sig| sig.inputs_and_output.split_last())
+                .map(|(output, inputs)| {
+                    output.is_unit()
+                        && inputs
+                            .get(0)
+                            .and_then(|self_ty| self_ty.ref_mutability())
+                            .map_or(false, rustc_ast::Mutability::is_mut)
+                })
+                .unwrap_or(false)
+        };
+
+        if !(rcvr_has_the_expected_type || prev_call_mutates_and_returns_unit()) {
+            return;
+        }
+
+        let mut sp = MultiSpan::from_span(path_segment.ident.span);
+        sp.push_span_label(
+            path_segment.ident.span,
+            format!(
+                "this call modifies {} in-place",
+                match rcvr.kind {
+                    ExprKind::Path(QPath::Resolved(
+                        None,
+                        hir::Path { segments: [segment], .. },
+                    )) => format!("`{}`", segment.ident),
+                    _ => "its receiver".to_string(),
+                }
+            ),
+        );
+
+        let modifies_rcvr_note =
+            format!("method `{}` modifies its receiver in-place", path_segment.ident);
+        if rcvr_has_the_expected_type {
             sp.push_span_label(
                 rcvr.span,
                 "you probably want to use this value after calling the method...",
             );
+            err.span_note(sp, &modifies_rcvr_note);
+            err.note(&format!("...instead of the `()` output of method `{}`", path_segment.ident));
+        } else if let ExprKind::MethodCall(..) = rcvr.kind {
             err.span_note(
                 sp,
-                &format!("method `{}` modifies its receiver in-place", path_segment.ident),
+                modifies_rcvr_note.clone() + ", it is not meant to be used in method chains.",
             );
-            err.note(&format!("...instead of the `()` output of method `{}`", path_segment.ident));
+        } else {
+            err.span_note(sp, &modifies_rcvr_note);
         }
     }
 
@@ -1319,7 +1365,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // This also occurs for an enum variant on a type alias.
             let impl_ty = self.normalize(span, tcx.type_of(impl_def_id).subst(tcx, substs));
             let self_ty = self.normalize(span, self_ty);
-            match self.at(&self.misc(span), self.param_env).eq(impl_ty, self_ty) {
+            match self.at(&self.misc(span), self.param_env).eq(
+                DefineOpaqueTypes::No,
+                impl_ty,
+                self_ty,
+            ) {
                 Ok(ok) => self.register_infer_ok_obligations(ok),
                 Err(_) => {
                     self.tcx.sess.delay_span_bug(
@@ -1367,41 +1417,19 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) {
         let param_env = self.param_env;
 
-        let remap = match self.tcx.def_kind(def_id) {
-            // Associated consts have `Self: ~const Trait` bounds that should be satisfiable when
-            // `Self: Trait` is satisfied because it does not matter whether the impl is `const`.
-            // Therefore we have to remap the param env here to be non-const.
-            hir::def::DefKind::AssocConst => true,
-            hir::def::DefKind::AssocFn
-                if self.tcx.def_kind(self.tcx.parent(def_id)) == hir::def::DefKind::Trait =>
-            {
-                // N.B.: All callsites to this function involve checking a path expression.
-                //
-                // When instantiating a trait method as a function item, it does not actually matter whether
-                // the trait is `const` or not, or whether `where T: ~const Tr` needs to be satisfied as
-                // `const`. If we were to introduce instantiating trait methods as `const fn`s, we would
-                // check that after this, either via a bound `where F: ~const FnOnce` or when coercing to a
-                // `const fn` pointer.
-                //
-                // FIXME(fee1-dead) FIXME(const_trait_impl): update this doc when trait methods can satisfy
-                // `~const FnOnce` or can be coerced to `const fn` pointer.
-                true
-            }
-            _ => false,
-        };
-        let (bounds, _) = self.instantiate_bounds(span, def_id, &substs);
+        let bounds = self.instantiate_bounds(span, def_id, &substs);
 
-        for mut obligation in traits::predicates_for_generics(
+        for obligation in traits::predicates_for_generics(
             |idx, predicate_span| {
                 traits::ObligationCause::new(span, self.body_id, code(idx, predicate_span))
             },
             param_env,
             bounds,
         ) {
-            if remap {
-                obligation = obligation.without_const(self.tcx);
-            }
-            self.register_predicate(obligation);
+            // N.B. We are remapping all predicates to non-const since we don't know if we just
+            // want them as function pointers or we are calling them from a const-context. The
+            // actual checking will occur in `rustc_const_eval::transform::check_consts`.
+            self.register_predicate(obligation.without_const(self.tcx));
         }
     }
 
@@ -1415,7 +1443,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         } else {
             let e = self.tainted_by_errors().unwrap_or_else(|| {
                 self.err_ctxt()
-                    .emit_inference_failure_err((**self).body_id, sp, ty.into(), E0282, true)
+                    .emit_inference_failure_err(self.body_id, sp, ty.into(), E0282, true)
                     .emit()
             });
             let err = self.tcx.ty_error(e);
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
index b09886fe3..f879ccbb3 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
@@ -3,10 +3,8 @@ use rustc_hir as hir;
 use rustc_hir::def::Res;
 use rustc_hir::def_id::DefId;
 use rustc_infer::traits::ObligationCauseCode;
-use rustc_middle::ty::{
-    self, DefIdTree, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor,
-};
-use rustc_span::{self, Span};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor};
+use rustc_span::{self, symbol::kw, Span};
 use rustc_trait_selection::traits;
 
 use std::ops::ControlFlow;
@@ -27,17 +25,28 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let generics = self.tcx.generics_of(def_id);
         let predicate_substs = match unsubstituted_pred.kind().skip_binder() {
-            ty::PredicateKind::Clause(ty::Clause::Trait(pred)) => pred.trait_ref.substs,
-            ty::PredicateKind::Clause(ty::Clause::Projection(pred)) => pred.projection_ty.substs,
-            _ => ty::List::empty(),
+            ty::PredicateKind::Clause(ty::Clause::Trait(pred)) => pred.trait_ref.substs.to_vec(),
+            ty::PredicateKind::Clause(ty::Clause::Projection(pred)) => {
+                pred.projection_ty.substs.to_vec()
+            }
+            ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(arg, ty)) => {
+                vec![ty.into(), arg.into()]
+            }
+            ty::PredicateKind::ConstEvaluatable(e) => vec![e.into()],
+            _ => return false,
         };
 
-        let find_param_matching = |matches: &dyn Fn(&ty::ParamTy) -> bool| {
-            predicate_substs.types().find_map(|ty| {
-                ty.walk().find_map(|arg| {
+        let find_param_matching = |matches: &dyn Fn(ty::ParamTerm) -> bool| {
+            predicate_substs.iter().find_map(|arg| {
+                arg.walk().find_map(|arg| {
                     if let ty::GenericArgKind::Type(ty) = arg.unpack()
-                        && let ty::Param(param_ty) = ty.kind()
-                        && matches(param_ty)
+                        && let ty::Param(param_ty) = *ty.kind()
+                        && matches(ty::ParamTerm::Ty(param_ty))
+                    {
+                        Some(arg)
+                    } else if let ty::GenericArgKind::Const(ct) = arg.unpack()
+                        && let ty::ConstKind::Param(param_ct) = ct.kind()
+                        && matches(ty::ParamTerm::Const(param_ct))
                     {
                         Some(arg)
                     } else {
@@ -49,26 +58,27 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // Prefer generics that are local to the fn item, since these are likely
         // to be the cause of the unsatisfied predicate.
-        let mut param_to_point_at = find_param_matching(&|param_ty| {
-            self.tcx.parent(generics.type_param(param_ty, self.tcx).def_id) == def_id
+        let mut param_to_point_at = find_param_matching(&|param_term| {
+            self.tcx.parent(generics.param_at(param_term.index(), self.tcx).def_id) == def_id
         });
         // Fall back to generic that isn't local to the fn item. This will come
         // from a trait or impl, for example.
-        let mut fallback_param_to_point_at = find_param_matching(&|param_ty| {
-            self.tcx.parent(generics.type_param(param_ty, self.tcx).def_id) != def_id
-                && param_ty.name != rustc_span::symbol::kw::SelfUpper
+        let mut fallback_param_to_point_at = find_param_matching(&|param_term| {
+            self.tcx.parent(generics.param_at(param_term.index(), self.tcx).def_id) != def_id
+                && !matches!(param_term, ty::ParamTerm::Ty(ty) if ty.name == kw::SelfUpper)
         });
         // Finally, the `Self` parameter is possibly the reason that the predicate
         // is unsatisfied. This is less likely to be true for methods, because
         // method probe means that we already kinda check that the predicates due
         // to the `Self` type are true.
-        let mut self_param_to_point_at =
-            find_param_matching(&|param_ty| param_ty.name == rustc_span::symbol::kw::SelfUpper);
+        let mut self_param_to_point_at = find_param_matching(
+            &|param_term| matches!(param_term, ty::ParamTerm::Ty(ty) if ty.name == kw::SelfUpper),
+        );
 
         // Finally, for ambiguity-related errors, we actually want to look
         // for a parameter that is the source of the inference type left
         // over in this predicate.
-        if let traits::FulfillmentErrorCode::CodeAmbiguity = error.code {
+        if let traits::FulfillmentErrorCode::CodeAmbiguity { .. } = error.code {
             fallback_param_to_point_at = None;
             self_param_to_point_at = None;
             param_to_point_at =
@@ -227,15 +237,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             .own_substs(ty::InternalSubsts::identity_for_item(self.tcx, def_id));
         let Some((index, _)) = own_substs
             .iter()
-            .filter(|arg| matches!(arg.unpack(), ty::GenericArgKind::Type(_)))
             .enumerate()
             .find(|(_, arg)| **arg == param_to_point_at) else { return false };
         let Some(arg) = segment
             .args()
             .args
-            .iter()
-            .filter(|arg| matches!(arg, hir::GenericArg::Type(_)))
-            .nth(index) else { return false; };
+            .get(index) else { return false; };
         error.obligation.cause.span = arg
             .span()
             .find_ancestor_in_same_ctxt(error.obligation.cause.span)
@@ -302,7 +309,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             .iter()
             .filter(|field| {
                 let field_ty = field.ty(self.tcx, identity_substs);
-                Self::find_param_in_ty(field_ty.into(), param_to_point_at)
+                find_param_in_ty(field_ty.into(), param_to_point_at)
             })
             .collect();
 
@@ -348,7 +355,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             .inputs()
             .iter()
             .enumerate()
-            .filter(|(_, ty)| Self::find_param_in_ty((**ty).into(), param_to_point_at))
+            .filter(|(_, ty)| find_param_in_ty((**ty).into(), param_to_point_at))
             .collect();
         // If there's one field that references the given generic, great!
         if let [(idx, _)] = args_referencing_param.as_slice()
@@ -459,7 +466,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// obligation. Hence we refine the `expr` "outwards-in" and bail at the first kind of expression/impl we don't recognize.
     ///
     /// This function returns a `Result<&Expr, &Expr>` - either way, it returns the `Expr` whose span should be
-    /// reported as an error. If it is `Ok`, then it means it refined successfull. If it is `Err`, then it may be
+    /// reported as an error. If it is `Ok`, then it means it refined successful. If it is `Err`, then it may be
     /// only a partial success - but it cannot be refined even further.
     fn blame_specific_expr_if_possible_for_derived_predicate_obligation(
         &self,
@@ -527,7 +534,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// - in_ty: `(Option<Vec<T>, bool)`
     /// we would drill until we arrive at `vec![1, 2, 3]`.
     ///
-    /// If successful, we return `Ok(refined_expr)`. If unsuccesful, we return `Err(partially_refined_expr`),
+    /// If successful, we return `Ok(refined_expr)`. If unsuccessful, we return `Err(partially_refined_expr`),
     /// which will go as far as possible. For example, given `(foo(), false)` instead, we would drill to
     /// `foo()` and then return `Err("foo()")`.
     ///
@@ -571,8 +578,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Find out which of `in_ty_elements` refer to `param`.
             // FIXME: It may be better to take the first if there are multiple,
             // just so that the error points to a smaller expression.
-            let Some((drill_expr, drill_ty)) = Self::is_iterator_singleton(expr_elements.iter().zip( in_ty_elements.iter()).filter(|(_expr_elem, in_ty_elem)| {
-                Self::find_param_in_ty((*in_ty_elem).into(), param)
+            let Some((drill_expr, drill_ty)) = is_iterator_singleton(expr_elements.iter().zip( in_ty_elements.iter()).filter(|(_expr_elem, in_ty_elem)| {
+                find_param_in_ty((*in_ty_elem).into(), param)
             })) else {
                 // The param is not mentioned, or it is mentioned in multiple indexes.
                 return Err(expr);
@@ -620,10 +627,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // We need to know which of the generic parameters mentions our target param.
             // We expect that at least one of them does, since it is expected to be mentioned.
             let Some((drill_generic_index, generic_argument_type)) =
-                Self::is_iterator_singleton(
+                is_iterator_singleton(
                     in_ty_adt_generic_args.iter().enumerate().filter(
                         |(_index, in_ty_generic)| {
-                            Self::find_param_in_ty(*in_ty_generic, param)
+                            find_param_in_ty(*in_ty_generic, param)
                         },
                     ),
                 ) else {
@@ -743,10 +750,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // We need to know which of the generic parameters mentions our target param.
             // We expect that at least one of them does, since it is expected to be mentioned.
             let Some((drill_generic_index, generic_argument_type)) =
-                Self::is_iterator_singleton(
+                is_iterator_singleton(
                     in_ty_adt_generic_args.iter().enumerate().filter(
                         |(_index, in_ty_generic)| {
-                            Self::find_param_in_ty(*in_ty_generic, param)
+                            find_param_in_ty(*in_ty_generic, param)
                         },
                     ),
                 ) else {
@@ -785,14 +792,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             //     outer contextual information.
 
             // (1) Find the (unique) field index which mentions the type in our constraint:
-            let Some((field_index, field_type)) = Self::is_iterator_singleton(
+            let Some((field_index, field_type)) = is_iterator_singleton(
                 in_ty_adt
                     .variant_with_id(variant_def_id)
                     .fields
                     .iter()
                     .map(|field| field.ty(self.tcx, *in_ty_adt_generic_args))
                     .enumerate()
-                    .filter(|(_index, field_type)| Self::find_param_in_ty((*field_type).into(), param))
+                    .filter(|(_index, field_type)| find_param_in_ty((*field_type).into(), param))
             ) else {
                 return Err(expr);
             };
@@ -825,20 +832,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         Err(expr)
     }
+}
 
-    // FIXME: This can be made into a private, non-impl function later.
-    /// Traverses the given ty (either a `ty::Ty` or a `ty::GenericArg`) and searches for references
-    /// to the given `param_to_point_at`. Returns `true` if it finds any use of the param.
-    pub fn find_param_in_ty(
-        ty: ty::GenericArg<'tcx>,
-        param_to_point_at: ty::GenericArg<'tcx>,
-    ) -> bool {
-        let mut walk = ty.walk();
-        while let Some(arg) = walk.next() {
-            if arg == param_to_point_at {
-                return true;
-            }
-            if let ty::GenericArgKind::Type(ty) = arg.unpack()
+/// Traverses the given ty (either a `ty::Ty` or a `ty::GenericArg`) and searches for references
+/// to the given `param_to_point_at`. Returns `true` if it finds any use of the param.
+fn find_param_in_ty<'tcx>(
+    ty: ty::GenericArg<'tcx>,
+    param_to_point_at: ty::GenericArg<'tcx>,
+) -> bool {
+    let mut walk = ty.walk();
+    while let Some(arg) = walk.next() {
+        if arg == param_to_point_at {
+            return true;
+        }
+        if let ty::GenericArgKind::Type(ty) = arg.unpack()
                 && let ty::Alias(ty::Projection, ..) = ty.kind()
             {
                 // This logic may seem a bit strange, but typically when
@@ -849,16 +856,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // in some UI tests.
                 walk.skip_current_subtree();
             }
-        }
-        false
     }
+    false
+}
 
-    // FIXME: This can be made into a private, non-impl function later.
-    /// Returns `Some(iterator.next())` if it has exactly one item, and `None` otherwise.
-    pub fn is_iterator_singleton<T>(mut iterator: impl Iterator<Item = T>) -> Option<T> {
-        match (iterator.next(), iterator.next()) {
-            (_, Some(_)) => None,
-            (first, _) => first,
-        }
+/// Returns `Some(iterator.next())` if it has exactly one item, and `None` otherwise.
+fn is_iterator_singleton<T>(mut iterator: impl Iterator<Item = T>) -> Option<T> {
+    match (iterator.next(), iterator.next()) {
+        (_, Some(_)) => None,
+        (first, _) => first,
     }
 }
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
index a46bdeb41..ea1b52daa 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
@@ -24,14 +24,14 @@ use rustc_hir_analysis::structured_errors::StructuredDiagnostic;
 use rustc_index::vec::IndexVec;
 use rustc_infer::infer::error_reporting::{FailureCode, ObligationCauseExt};
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use rustc_infer::infer::InferOk;
 use rustc_infer::infer::TypeTrace;
+use rustc_infer::infer::{DefineOpaqueTypes, InferOk};
 use rustc_middle::ty::adjustment::AllowTwoPhase;
 use rustc_middle::ty::visit::TypeVisitableExt;
-use rustc_middle::ty::{self, DefIdTree, IsSuggestable, Ty};
+use rustc_middle::ty::{self, IsSuggestable, Ty};
 use rustc_session::Session;
 use rustc_span::symbol::{kw, Ident};
-use rustc_span::{self, sym, Span};
+use rustc_span::{self, sym, BytePos, Span};
 use rustc_trait_selection::traits::{self, ObligationCauseCode, SelectionContext};
 
 use std::iter;
@@ -301,9 +301,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             // 3. Check if the formal type is a supertype of the checked one
             //    and register any such obligations for future type checks
-            let supertype_error = self
-                .at(&self.misc(provided_arg.span), self.param_env)
-                .sup(formal_input_ty, coerced_ty);
+            let supertype_error = self.at(&self.misc(provided_arg.span), self.param_env).sup(
+                DefineOpaqueTypes::No,
+                formal_input_ty,
+                coerced_ty,
+            );
             let subtyping_error = match supertype_error {
                 Ok(InferOk { obligations, value: () }) => {
                     self.register_predicates(obligations);
@@ -470,7 +472,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         err_code: &str,
         fn_def_id: Option<DefId>,
         call_span: Span,
-        call_expr: &hir::Expr<'tcx>,
+        call_expr: &'tcx hir::Expr<'tcx>,
     ) {
         // Next, let's construct the error
         let (error_span, full_call_span, call_name, is_method) = match &call_expr.kind {
@@ -585,7 +587,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             // Using probe here, since we don't want this subtyping to affect inference.
             let subtyping_error = self.probe(|_| {
-                self.at(&self.misc(arg_span), self.param_env).sup(formal_input_ty, coerced_ty).err()
+                self.at(&self.misc(arg_span), self.param_env)
+                    .sup(DefineOpaqueTypes::No, formal_input_ty, coerced_ty)
+                    .err()
             });
 
             // Same as above: if either the coerce type or the checked type is an error type,
@@ -764,7 +768,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let (provided_ty, provided_span) = provided_arg_tys[*provided_idx];
             let trace =
                 mk_trace(provided_span, formal_and_expected_inputs[*expected_idx], provided_ty);
-            if !matches!(trace.cause.as_failure_code(*e), FailureCode::Error0308(_)) {
+            if !matches!(trace.cause.as_failure_code(*e), FailureCode::Error0308) {
                 self.err_ctxt().report_and_explain_type_error(trace, *e).emit();
                 return true;
             }
@@ -803,24 +807,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 full_call_span,
                 format!("arguments to this {} are incorrect", call_name),
             );
-            if let (Some(callee_ty), hir::ExprKind::MethodCall(_, rcvr, _, _)) =
-                (callee_ty, &call_expr.kind)
+
+            if let hir::ExprKind::MethodCall(_, rcvr, _, _) = call_expr.kind
+                && provided_idx.as_usize() == expected_idx.as_usize()
             {
-                // Type that would have accepted this argument if it hadn't been inferred earlier.
-                // FIXME: We leave an inference variable for now, but it'd be nice to get a more
-                // specific type to increase the accuracy of the diagnostic.
-                let expected = self.infcx.next_ty_var(TypeVariableOrigin {
-                    kind: TypeVariableOriginKind::MiscVariable,
-                    span: full_call_span,
-                });
-                self.point_at_expr_source_of_inferred_type(
+                self.note_source_of_type_mismatch_constraint(
                     &mut err,
                     rcvr,
-                    expected,
-                    callee_ty,
-                    provided_arg_span,
+                    crate::demand::TypeMismatchSource::Arg {
+                        call_expr,
+                        incompatible_arg: provided_idx.as_usize(),
+                    },
                 );
             }
+
             // Call out where the function is defined
             self.label_fn_like(
                 &mut err,
@@ -890,8 +890,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         };
 
         let mut errors = errors.into_iter().peekable();
+        let mut only_extras_so_far = errors
+            .peek()
+            .map_or(false, |first| matches!(first, Error::Extra(arg_idx) if arg_idx.index() == 0));
         let mut suggestions = vec![];
         while let Some(error) = errors.next() {
+            only_extras_so_far &= matches!(error, Error::Extra(_));
+
             match error {
                 Error::Invalid(provided_idx, expected_idx, compatibility) => {
                     let (formal_ty, expected_ty) = formal_and_expected_inputs[expected_idx];
@@ -937,10 +942,34 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         if arg_idx.index() > 0
                         && let Some((_, prev)) = provided_arg_tys
                             .get(ProvidedIdx::from_usize(arg_idx.index() - 1)
-                    ) {
-                        // Include previous comma
-                        span = prev.shrink_to_hi().to(span);
-                    }
+                        ) {
+                            // Include previous comma
+                            span = prev.shrink_to_hi().to(span);
+                        }
+
+                        // Is last argument for deletion in a row starting from the 0-th argument?
+                        // Then delete the next comma, so we are not left with `f(, ...)`
+                        //
+                        //     fn f() {}
+                        //   - f(0, 1,)
+                        //   + f()
+                        if only_extras_so_far
+                            && errors
+                                .peek()
+                                .map_or(true, |next_error| !matches!(next_error, Error::Extra(_)))
+                        {
+                            let next = provided_arg_tys
+                                .get(arg_idx + 1)
+                                .map(|&(_, sp)| sp)
+                                .unwrap_or_else(|| {
+                                    // Subtract one to move before `)`
+                                    call_expr.span.with_lo(call_expr.span.hi() - BytePos(1))
+                                });
+
+                            // Include next comma
+                            span = span.until(next);
+                        }
+
                         suggestions.push((span, String::new()));
 
                         suggestion_text = match suggestion_text {
@@ -1158,11 +1187,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // index position where it *should have been*, which is *after* the previous one.
             if let Some(provided_idx) = provided_idx {
                 prev = provided_idx.index() as i64;
+                continue;
             }
             let idx = ProvidedIdx::from_usize((prev + 1) as usize);
-            if let None = provided_idx
-                && let Some((_, arg_span)) = provided_arg_tys.get(idx)
-            {
+            if let Some((_, arg_span)) = provided_arg_tys.get(idx) {
+                prev += 1;
                 // There is a type that was *not* found anywhere, so it isn't a move, but a
                 // replacement and we look at what type it should have been. This will allow us
                 // To suggest a multipart suggestion when encountering `foo(1, "")` where the def
@@ -1380,7 +1409,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             self.demand_eqtype(init.span, local_ty, init_ty);
             init_ty
         } else {
-            self.check_expr_coercable_to_type(init, local_ty, None)
+            self.check_expr_coercible_to_type(init, local_ty, None)
         }
     }
 
@@ -1665,7 +1694,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// Given a function block's `HirId`, returns its `FnDecl` if it exists, or `None` otherwise.
     fn get_parent_fn_decl(&self, blk_id: hir::HirId) -> Option<(&'tcx hir::FnDecl<'tcx>, Ident)> {
         let parent = self.tcx.hir().get_by_def_id(self.tcx.hir().get_parent_item(blk_id).def_id);
-        self.get_node_fn_decl(parent).map(|(fn_decl, ident, _)| (fn_decl, ident))
+        self.get_node_fn_decl(parent).map(|(_, fn_decl, ident, _)| (fn_decl, ident))
     }
 
     /// If `expr` is a `match` expression that has only one non-`!` arm, use that arm's tail
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
index 1dea3e6f9..c6fd0b610 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
@@ -211,13 +211,13 @@ impl<'a, 'tcx> AstConv<'tcx> for FnCtxt<'a, 'tcx> {
     fn get_type_parameter_bounds(
         &self,
         _: Span,
-        def_id: DefId,
+        def_id: LocalDefId,
         _: Ident,
     ) -> ty::GenericPredicates<'tcx> {
         let tcx = self.tcx;
-        let item_def_id = tcx.hir().ty_param_owner(def_id.expect_local());
+        let item_def_id = tcx.hir().ty_param_owner(def_id);
         let generics = tcx.generics_of(item_def_id);
-        let index = generics.param_def_id_to_index[&def_id];
+        let index = generics.param_def_id_to_index[&def_id.to_def_id()];
         ty::GenericPredicates {
             parent: None,
             predicates: tcx.arena.alloc_from_iter(
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
index c49621b7c..5fda4e191 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
@@ -16,7 +16,7 @@ use rustc_infer::traits::{self, StatementAsExpression};
 use rustc_middle::lint::in_external_macro;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{
-    self, suggest_constraining_type_params, Binder, DefIdTree, IsSuggestable, ToPredicate, Ty,
+    self, suggest_constraining_type_params, Binder, IsSuggestable, ToPredicate, Ty,
     TypeVisitableExt,
 };
 use rustc_session::errors::ExprParenthesesNeeded;
@@ -64,8 +64,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let expr = expr.peel_drop_temps();
         self.suggest_missing_semicolon(err, expr, expected, false);
         let mut pointing_at_return_type = false;
-        if let Some((fn_decl, can_suggest)) = self.get_fn_decl(blk_id) {
-            let fn_id = self.tcx.hir().get_return_block(blk_id).unwrap();
+        if let Some((fn_id, fn_decl, can_suggest)) = self.get_fn_decl(blk_id) {
             pointing_at_return_type = self.suggest_missing_return_type(
                 err,
                 &fn_decl,
@@ -166,8 +165,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         ty: Ty<'tcx>,
     ) -> Option<(DefIdOrName, Ty<'tcx>, Vec<Ty<'tcx>>)> {
-        let body_hir_id = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
-        self.err_ctxt().extract_callable_info(body_hir_id, self.param_env, ty)
+        self.err_ctxt().extract_callable_info(self.body_id, self.param_env, ty)
     }
 
     pub fn suggest_two_fn_call(
@@ -669,6 +667,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// This routine checks if the return type is left as default, the method is not part of an
     /// `impl` block and that it isn't the `main` method. If so, it suggests setting the return
     /// type.
+    #[instrument(level = "trace", skip(self, err))]
     pub(in super::super) fn suggest_missing_return_type(
         &self,
         err: &mut Diagnostic,
@@ -705,28 +704,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     return true
                 }
             }
-            hir::FnRetTy::Return(ty) => {
-                let span = ty.span;
-
-                if let hir::TyKind::OpaqueDef(item_id, ..) = ty.kind
-                && let hir::Node::Item(hir::Item {
-                    kind: hir::ItemKind::OpaqueTy(op_ty),
-                    ..
-                }) = self.tcx.hir().get(item_id.hir_id())
-                && let hir::OpaqueTy {
-                    bounds: [bound], ..
-                } = op_ty
-                && let hir::GenericBound::LangItemTrait(
-                    hir::LangItem::Future, _, _, generic_args) = bound
-                && let hir::GenericArgs { bindings: [ty_binding], .. } = generic_args
-                && let hir::TypeBinding { kind, .. } = ty_binding
-                && let hir::TypeBindingKind::Equality { term } = kind
-                && let hir::Term::Ty(term_ty) = term {
+            hir::FnRetTy::Return(hir_ty) => {
+                let span = hir_ty.span;
+
+                if let hir::TyKind::OpaqueDef(item_id, ..) = hir_ty.kind
+                    && let hir::Node::Item(hir::Item {
+                        kind: hir::ItemKind::OpaqueTy(op_ty),
+                        ..
+                    }) = self.tcx.hir().get(item_id.hir_id())
+                    && let [hir::GenericBound::LangItemTrait(
+                        hir::LangItem::Future, _, _, generic_args)] = op_ty.bounds
+                    && let hir::GenericArgs { bindings: [ty_binding], .. } = generic_args
+                    && let hir::TypeBindingKind::Equality { term: hir::Term::Ty(term) } = ty_binding.kind
+                {
                     // Check if async function's return type was omitted.
                     // Don't emit suggestions if the found type is `impl Future<...>`.
-                    debug!("suggest_missing_return_type: found = {:?}", found);
+                    debug!(?found);
                     if found.is_suggestable(self.tcx, false) {
-                        if term_ty.span.is_empty() {
+                        if term.span.is_empty() {
                             err.subdiagnostic(AddReturnTypeSuggestion::Add { span, found: found.to_string() });
                             return true;
                         } else {
@@ -737,11 +732,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                 // Only point to return type if the expected type is the return type, as if they
                 // are not, the expectation must have been caused by something else.
-                debug!("suggest_missing_return_type: return type {:?} node {:?}", ty, ty.kind);
-                let ty = self.astconv().ast_ty_to_ty(ty);
-                debug!("suggest_missing_return_type: return type {:?}", ty);
-                debug!("suggest_missing_return_type: expected type {:?}", ty);
-                let bound_vars = self.tcx.late_bound_vars(fn_id);
+                debug!("return type {:?}", hir_ty);
+                let ty = self.astconv().ast_ty_to_ty(hir_ty);
+                debug!("return type {:?}", ty);
+                debug!("expected type {:?}", expected);
+                let bound_vars = self.tcx.late_bound_vars(hir_ty.hir_id.owner.into());
                 let ty = Binder::bind_with_vars(ty, bound_vars);
                 let ty = self.normalize(span, ty);
                 let ty = self.tcx.erase_late_bound_regions(ty);
@@ -988,13 +983,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 )
                 .must_apply_modulo_regions()
           {
-              diag.span_suggestion_verbose(
-                  expr.span.shrink_to_hi(),
-                  "consider using clone here",
-                  ".clone()",
-                  Applicability::MachineApplicable,
-              );
-              return true;
+            let suggestion = match self.maybe_get_struct_pattern_shorthand_field(expr) {
+                Some(ident) => format!(": {}.clone()", ident),
+                None => ".clone()".to_string()
+            };
+
+            diag.span_suggestion_verbose(
+                expr.span.shrink_to_hi(),
+                "consider using clone here",
+                suggestion,
+                Applicability::MachineApplicable,
+            );
+            return true;
           }
         false
     }
@@ -1015,11 +1015,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let mut suggest_copied_or_cloned = || {
             let expr_inner_ty = substs.type_at(0);
             let expected_inner_ty = expected_substs.type_at(0);
-            if let ty::Ref(_, ty, hir::Mutability::Not) = expr_inner_ty.kind()
-                && self.can_eq(self.param_env, *ty, expected_inner_ty)
+            if let &ty::Ref(_, ty, hir::Mutability::Not) = expr_inner_ty.kind()
+                && self.can_eq(self.param_env, ty, expected_inner_ty)
             {
                 let def_path = self.tcx.def_path_str(adt_def.did());
-                if self.type_is_copy_modulo_regions(self.param_env, *ty, expr.span) {
+                if self.type_is_copy_modulo_regions(self.param_env, ty) {
                     diag.span_suggestion_verbose(
                         expr.span.shrink_to_hi(),
                         format!(
@@ -1033,9 +1033,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     && rustc_trait_selection::traits::type_known_to_meet_bound_modulo_regions(
                         self,
                         self.param_env,
-                        *ty,
+                        ty,
                         clone_did,
-                        expr.span
                     )
                 {
                     diag.span_suggestion_verbose(
@@ -1156,13 +1155,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return false;
         }
 
-        diag.span_suggestion(
+        let suggestion = match self.maybe_get_struct_pattern_shorthand_field(expr) {
+            Some(ident) => format!(": {}.is_some()", ident),
+            None => ".is_some()".to_string(),
+        };
+
+        diag.span_suggestion_verbose(
             expr.span.shrink_to_hi(),
             "use `Option::is_some` to test if the `Option` has a value",
-            ".is_some()",
+            suggestion,
             Applicability::MachineApplicable,
         );
-
         true
     }
 
diff --git a/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/cfg_build.rs b/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/cfg_build.rs
index 7c0402b1c..3e9a9ce1b 100644
--- a/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/cfg_build.rs
+++ b/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/cfg_build.rs
@@ -190,7 +190,6 @@ impl<'a, 'tcx> DropRangeVisitor<'a, 'tcx> {
             //
             // Some of these may be interesting in the future
             ExprKind::Path(..)
-            | ExprKind::Box(..)
             | ExprKind::ConstBlock(..)
             | ExprKind::Array(..)
             | ExprKind::Call(..)
@@ -478,7 +477,6 @@ impl<'a, 'tcx> Visitor<'tcx> for DropRangeVisitor<'a, 'tcx> {
             | ExprKind::AssignOp(..)
             | ExprKind::Binary(..)
             | ExprKind::Block(..)
-            | ExprKind::Box(..)
             | ExprKind::Cast(..)
             | ExprKind::Closure { .. }
             | ExprKind::ConstBlock(..)
@@ -528,8 +526,9 @@ impl DropRangesBuilder {
         let mut next = <_>::from(0u32);
         for value in tracked_values {
             for_each_consumable(hir, value, |value| {
-                if !tracked_value_map.contains_key(&value) {
-                    tracked_value_map.insert(value, next);
+                if let std::collections::hash_map::Entry::Vacant(e) = tracked_value_map.entry(value)
+                {
+                    e.insert(next);
                     next = next + 1;
                 }
             });
diff --git a/compiler/rustc_hir_typeck/src/generator_interior/mod.rs b/compiler/rustc_hir_typeck/src/generator_interior/mod.rs
index 2e41c2041..f39710804 100644
--- a/compiler/rustc_hir_typeck/src/generator_interior/mod.rs
+++ b/compiler/rustc_hir_typeck/src/generator_interior/mod.rs
@@ -13,7 +13,7 @@ use rustc_hir::def_id::DefId;
 use rustc_hir::hir_id::HirIdSet;
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{Arm, Expr, ExprKind, Guard, HirId, Pat, PatKind};
-use rustc_infer::infer::RegionVariableOrigin;
+use rustc_infer::infer::{DefineOpaqueTypes, RegionVariableOrigin};
 use rustc_middle::middle::region::{self, Scope, ScopeData, YieldData};
 use rustc_middle::ty::fold::FnMutDelegate;
 use rustc_middle::ty::{self, BoundVariableKind, RvalueScopes, Ty, TyCtxt, TypeVisitableExt};
@@ -239,8 +239,7 @@ pub fn resolve_interior<'a, 'tcx>(
             // typeck had previously found constraints that would cause them to be related.
 
             let mut counter = 0;
-            let mut mk_bound_region = |span| {
-                let kind = ty::BrAnon(counter, span);
+            let mut mk_bound_region = |kind| {
                 let var = ty::BoundVar::from_u32(counter);
                 counter += 1;
                 ty::BoundRegion { var, kind }
@@ -252,24 +251,23 @@ pub fn resolve_interior<'a, 'tcx>(
                         let origin = fcx.region_var_origin(vid);
                         match origin {
                             RegionVariableOrigin::EarlyBoundRegion(span, _) => {
-                                mk_bound_region(Some(span))
+                                mk_bound_region(ty::BrAnon(Some(span)))
                             }
-                            _ => mk_bound_region(None),
+                            _ => mk_bound_region(ty::BrAnon(None)),
                         }
                     }
-                    // FIXME: these should use `BrNamed`
                     ty::ReEarlyBound(region) => {
-                        mk_bound_region(Some(fcx.tcx.def_span(region.def_id)))
+                        mk_bound_region(ty::BrNamed(region.def_id, region.name))
                     }
                     ty::ReLateBound(_, ty::BoundRegion { kind, .. })
                     | ty::ReFree(ty::FreeRegion { bound_region: kind, .. }) => match kind {
-                        ty::BoundRegionKind::BrAnon(_, span) => mk_bound_region(span),
-                        ty::BoundRegionKind::BrNamed(def_id, _) => {
-                            mk_bound_region(Some(fcx.tcx.def_span(def_id)))
+                        ty::BoundRegionKind::BrAnon(span) => mk_bound_region(ty::BrAnon(span)),
+                        ty::BoundRegionKind::BrNamed(def_id, sym) => {
+                            mk_bound_region(ty::BrNamed(def_id, sym))
                         }
-                        ty::BoundRegionKind::BrEnv => mk_bound_region(None),
+                        ty::BoundRegionKind::BrEnv => mk_bound_region(ty::BrAnon(None)),
                     },
-                    _ => mk_bound_region(None),
+                    _ => mk_bound_region(ty::BrAnon(None)),
                 };
                 let r = fcx.tcx.mk_re_late_bound(current_depth, br);
                 r
@@ -293,10 +291,7 @@ pub fn resolve_interior<'a, 'tcx>(
             type_causes,
             FnMutDelegate {
                 regions: &mut |br| {
-                    let kind = match br.kind {
-                        ty::BrAnon(_, span) => ty::BrAnon(counter, span),
-                        _ => br.kind,
-                    };
+                    let kind = br.kind;
                     let var = ty::BoundVar::from_usize(bound_vars.len());
                     bound_vars.push(ty::BoundVariableKind::Region(kind));
                     counter += 1;
@@ -313,8 +308,7 @@ pub fn resolve_interior<'a, 'tcx>(
     // Extract type components to build the witness type.
     let type_list = fcx.tcx.mk_type_list_from_iter(type_causes.iter().map(|cause| cause.ty));
     let bound_vars = fcx.tcx.mk_bound_variable_kinds(&bound_vars);
-    let witness =
-        fcx.tcx.mk_generator_witness(ty::Binder::bind_with_vars(type_list, bound_vars.clone()));
+    let witness = fcx.tcx.mk_generator_witness(ty::Binder::bind_with_vars(type_list, bound_vars));
 
     drop(typeck_results);
     // Store the generator types and spans into the typeck results for this generator.
@@ -327,7 +321,11 @@ pub fn resolve_interior<'a, 'tcx>(
     );
 
     // Unify the type variable inside the generator with the new witness
-    match fcx.at(&fcx.misc(body.value.span), fcx.param_env).eq(interior, witness) {
+    match fcx.at(&fcx.misc(body.value.span), fcx.param_env).eq(
+        DefineOpaqueTypes::No,
+        interior,
+        witness,
+    ) {
         Ok(ok) => fcx.register_infer_ok_obligations(ok),
         _ => bug!("failed to relate {interior} and {witness}"),
     }
diff --git a/compiler/rustc_hir_typeck/src/inherited.rs b/compiler/rustc_hir_typeck/src/inherited.rs
index 26020382d..4110b176b 100644
--- a/compiler/rustc_hir_typeck/src/inherited.rs
+++ b/compiler/rustc_hir_typeck/src/inherited.rs
@@ -4,7 +4,6 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::HirIdMap;
-use rustc_infer::infer;
 use rustc_infer::infer::{DefiningAnchor, InferCtxt, InferOk, TyCtxtInferExt};
 use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -58,8 +57,6 @@ pub struct Inherited<'tcx> {
     pub(super) deferred_generator_interiors:
         RefCell<Vec<(LocalDefId, hir::BodyId, Ty<'tcx>, hir::GeneratorKind)>>,
 
-    pub(super) body_id: Option<hir::BodyId>,
-
     /// Whenever we introduce an adjustment from `!` into a type variable,
     /// we record that type variable here. This is later used to inform
     /// fallback. See the `fallback` module for details.
@@ -75,48 +72,16 @@ impl<'tcx> Deref for Inherited<'tcx> {
     }
 }
 
-/// A temporary returned by `Inherited::build(...)`. This is necessary
-/// for multiple `InferCtxt` to share the same `typeck_results`
-/// without using `Rc` or something similar.
-pub struct InheritedBuilder<'tcx> {
-    infcx: infer::InferCtxtBuilder<'tcx>,
-    def_id: LocalDefId,
-    typeck_results: RefCell<ty::TypeckResults<'tcx>>,
-}
-
 impl<'tcx> Inherited<'tcx> {
-    pub fn build(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> InheritedBuilder<'tcx> {
+    pub fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Self {
         let hir_owner = tcx.hir().local_def_id_to_hir_id(def_id).owner;
 
-        InheritedBuilder {
-            infcx: tcx
-                .infer_ctxt()
-                .ignoring_regions()
-                .with_opaque_type_inference(DefiningAnchor::Bind(hir_owner.def_id)),
-            def_id,
-            typeck_results: RefCell::new(ty::TypeckResults::new(hir_owner)),
-        }
-    }
-}
-
-impl<'tcx> InheritedBuilder<'tcx> {
-    pub fn enter<F, R>(mut self, f: F) -> R
-    where
-        F: FnOnce(&Inherited<'tcx>) -> R,
-    {
-        let def_id = self.def_id;
-        f(&Inherited::new(self.infcx.build(), def_id, self.typeck_results))
-    }
-}
-
-impl<'tcx> Inherited<'tcx> {
-    fn new(
-        infcx: InferCtxt<'tcx>,
-        def_id: LocalDefId,
-        typeck_results: RefCell<ty::TypeckResults<'tcx>>,
-    ) -> Self {
-        let tcx = infcx.tcx;
-        let body_id = tcx.hir().maybe_body_owned_by(def_id);
+        let infcx = tcx
+            .infer_ctxt()
+            .ignoring_regions()
+            .with_opaque_type_inference(DefiningAnchor::Bind(hir_owner.def_id))
+            .build();
+        let typeck_results = RefCell::new(ty::TypeckResults::new(hir_owner));
 
         Inherited {
             typeck_results,
@@ -130,7 +95,6 @@ impl<'tcx> Inherited<'tcx> {
             deferred_asm_checks: RefCell::new(Vec::new()),
             deferred_generator_interiors: RefCell::new(Vec::new()),
             diverging_type_vars: RefCell::new(Default::default()),
-            body_id,
             infer_var_info: RefCell::new(Default::default()),
         }
     }
diff --git a/compiler/rustc_hir_typeck/src/intrinsicck.rs b/compiler/rustc_hir_typeck/src/intrinsicck.rs
index 19d2befc4..106f5bcd7 100644
--- a/compiler/rustc_hir_typeck/src/intrinsicck.rs
+++ b/compiler/rustc_hir_typeck/src/intrinsicck.rs
@@ -4,7 +4,7 @@ use rustc_hir as hir;
 use rustc_index::vec::Idx;
 use rustc_middle::ty::layout::{LayoutError, SizeSkeleton};
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
-use rustc_target::abi::{Pointer, VariantIdx};
+use rustc_target::abi::{FieldIdx, Pointer, VariantIdx};
 
 use super::FnCtxt;
 
@@ -28,7 +28,7 @@ fn unpack_option_like<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Ty<'tcx> {
         }
 
         if def.variant(data_idx).fields.len() == 1 {
-            return def.variant(data_idx).fields[0].ty(tcx, substs);
+            return def.variant(data_idx).fields[FieldIdx::from_u32(0)].ty(tcx, substs);
         }
     }
 
@@ -84,6 +84,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let skeleton_string = |ty: Ty<'tcx>, sk| match sk {
             Ok(SizeSkeleton::Known(size)) => format!("{} bits", size.bits()),
             Ok(SizeSkeleton::Pointer { tail, .. }) => format!("pointer to `{tail}`"),
+            Ok(SizeSkeleton::Generic(size)) => {
+                if let Some(size) = size.try_eval_target_usize(tcx, self.param_env) {
+                    format!("{size} bytes")
+                } else {
+                    format!("generic size {size}")
+                }
+            }
             Err(LayoutError::Unknown(bad)) => {
                 if bad == ty {
                     "this type does not have a fixed size".to_owned()
diff --git a/compiler/rustc_hir_typeck/src/lib.rs b/compiler/rustc_hir_typeck/src/lib.rs
index e397dfd45..45890abad 100644
--- a/compiler/rustc_hir_typeck/src/lib.rs
+++ b/compiler/rustc_hir_typeck/src/lib.rs
@@ -45,13 +45,14 @@ mod rvalue_scopes;
 mod upvar;
 mod writeback;
 
-pub use diverges::Diverges;
-pub use expectation::Expectation;
-pub use fn_ctxt::*;
-pub use inherited::{Inherited, InheritedBuilder};
+pub use fn_ctxt::FnCtxt;
+pub use inherited::Inherited;
 
 use crate::check::check_fn;
 use crate::coercion::DynamicCoerceMany;
+use crate::diverges::Diverges;
+use crate::expectation::Expectation;
+use crate::fn_ctxt::RawTy;
 use crate::gather_locals::GatherLocalsVisitor;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::{
@@ -74,7 +75,7 @@ use rustc_session::Session;
 use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::{sym, Span};
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 #[macro_export]
 macro_rules! type_error_struct {
@@ -105,10 +106,9 @@ pub struct LocalTy<'tcx> {
 /// (notably closures), `typeck_results(def_id)` would wind up
 /// redirecting to the owning function.
 fn primary_body_of(
-    tcx: TyCtxt<'_>,
-    id: hir::HirId,
+    node: Node<'_>,
 ) -> Option<(hir::BodyId, Option<&hir::Ty<'_>>, Option<&hir::FnSig<'_>>)> {
-    match tcx.hir().get(id) {
+    match node {
         Node::Item(item) => match item.kind {
             hir::ItemKind::Const(ty, body) | hir::ItemKind::Static(ty, _, body) => {
                 Some((body, Some(ty), None))
@@ -142,8 +142,7 @@ fn has_typeck_results(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
     }
 
     if let Some(def_id) = def_id.as_local() {
-        let id = tcx.hir().local_def_id_to_hir_id(def_id);
-        primary_body_of(tcx, id).is_some()
+        primary_body_of(tcx.hir().get_by_def_id(def_id)).is_some()
     } else {
         false
     }
@@ -198,143 +197,140 @@ fn typeck_with_fallback<'tcx>(
     }
 
     let id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let node = tcx.hir().get(id);
     let span = tcx.hir().span(id);
 
     // Figure out what primary body this item has.
-    let (body_id, body_ty, fn_sig) = primary_body_of(tcx, id).unwrap_or_else(|| {
+    let (body_id, body_ty, fn_sig) = primary_body_of(node).unwrap_or_else(|| {
         span_bug!(span, "can't type-check body of {:?}", def_id);
     });
     let body = tcx.hir().body(body_id);
 
-    let typeck_results = Inherited::build(tcx, def_id).enter(|inh| {
-        let param_env = tcx.param_env(def_id);
-        let param_env = if tcx.has_attr(def_id.to_def_id(), sym::rustc_do_not_const_check) {
-            param_env.without_const()
+    let param_env = tcx.param_env(def_id);
+    let param_env = if tcx.has_attr(def_id, sym::rustc_do_not_const_check) {
+        param_env.without_const()
+    } else {
+        param_env
+    };
+    let inh = Inherited::new(tcx, def_id);
+    let mut fcx = FnCtxt::new(&inh, param_env, def_id);
+
+    if let Some(hir::FnSig { header, decl, .. }) = fn_sig {
+        let fn_sig = if rustc_hir_analysis::collect::get_infer_ret_ty(&decl.output).is_some() {
+            fcx.astconv().ty_of_fn(id, header.unsafety, header.abi, decl, None, None)
         } else {
-            param_env
+            tcx.fn_sig(def_id).subst_identity()
         };
-        let mut fcx = FnCtxt::new(&inh, param_env, def_id);
-
-        if let Some(hir::FnSig { header, decl, .. }) = fn_sig {
-            let fn_sig = if rustc_hir_analysis::collect::get_infer_ret_ty(&decl.output).is_some() {
-                fcx.astconv().ty_of_fn(id, header.unsafety, header.abi, decl, None, None)
-            } else {
-                tcx.fn_sig(def_id).subst_identity()
-            };
 
-            check_abi(tcx, id, span, fn_sig.abi());
+        check_abi(tcx, id, span, fn_sig.abi());
 
-            // Compute the function signature from point of view of inside the fn.
-            let fn_sig = tcx.liberate_late_bound_regions(def_id.to_def_id(), fn_sig);
-            let fn_sig = fcx.normalize(body.value.span, fn_sig);
+        // Compute the function signature from point of view of inside the fn.
+        let fn_sig = tcx.liberate_late_bound_regions(def_id.to_def_id(), fn_sig);
+        let fn_sig = fcx.normalize(body.value.span, fn_sig);
 
-            check_fn(&mut fcx, fn_sig, decl, def_id, body, None);
-        } else {
-            let expected_type = body_ty
-                .and_then(|ty| match ty.kind {
-                    hir::TyKind::Infer => Some(fcx.astconv().ast_ty_to_ty(ty)),
-                    _ => None,
-                })
-                .unwrap_or_else(|| match tcx.hir().get(id) {
-                    Node::AnonConst(_) => match tcx.hir().get(tcx.hir().parent_id(id)) {
-                        Node::Expr(&hir::Expr {
-                            kind: hir::ExprKind::ConstBlock(ref anon_const),
-                            ..
-                        }) if anon_const.hir_id == id => fcx.next_ty_var(TypeVariableOrigin {
-                            kind: TypeVariableOriginKind::TypeInference,
-                            span,
-                        }),
-                        Node::Ty(&hir::Ty {
-                            kind: hir::TyKind::Typeof(ref anon_const), ..
-                        }) if anon_const.hir_id == id => fcx.next_ty_var(TypeVariableOrigin {
-                            kind: TypeVariableOriginKind::TypeInference,
-                            span,
-                        }),
-                        Node::Expr(&hir::Expr { kind: hir::ExprKind::InlineAsm(asm), .. })
-                        | Node::Item(&hir::Item { kind: hir::ItemKind::GlobalAsm(asm), .. }) => {
-                            let operand_ty =
-                                asm.operands.iter().find_map(|(op, _op_sp)| match op {
-                                    hir::InlineAsmOperand::Const { anon_const }
-                                        if anon_const.hir_id == id =>
-                                    {
-                                        // Inline assembly constants must be integers.
-                                        Some(fcx.next_int_var())
-                                    }
-                                    hir::InlineAsmOperand::SymFn { anon_const }
-                                        if anon_const.hir_id == id =>
-                                    {
-                                        Some(fcx.next_ty_var(TypeVariableOrigin {
-                                            kind: TypeVariableOriginKind::MiscVariable,
-                                            span,
-                                        }))
-                                    }
-                                    _ => None,
-                                });
-                            operand_ty.unwrap_or_else(fallback)
+        check_fn(&mut fcx, fn_sig, decl, def_id, body, None);
+    } else {
+        let expected_type = if let Some(&hir::Ty { kind: hir::TyKind::Infer, span, .. }) = body_ty {
+            Some(fcx.next_ty_var(TypeVariableOrigin {
+                kind: TypeVariableOriginKind::TypeInference,
+                span,
+            }))
+        } else if let Node::AnonConst(_) = node {
+            match tcx.hir().get(tcx.hir().parent_id(id)) {
+                Node::Expr(&hir::Expr {
+                    kind: hir::ExprKind::ConstBlock(ref anon_const), ..
+                }) if anon_const.hir_id == id => Some(fcx.next_ty_var(TypeVariableOrigin {
+                    kind: TypeVariableOriginKind::TypeInference,
+                    span,
+                })),
+                Node::Ty(&hir::Ty { kind: hir::TyKind::Typeof(ref anon_const), .. })
+                    if anon_const.hir_id == id =>
+                {
+                    Some(fcx.next_ty_var(TypeVariableOrigin {
+                        kind: TypeVariableOriginKind::TypeInference,
+                        span,
+                    }))
+                }
+                Node::Expr(&hir::Expr { kind: hir::ExprKind::InlineAsm(asm), .. })
+                | Node::Item(&hir::Item { kind: hir::ItemKind::GlobalAsm(asm), .. }) => {
+                    asm.operands.iter().find_map(|(op, _op_sp)| match op {
+                        hir::InlineAsmOperand::Const { anon_const } if anon_const.hir_id == id => {
+                            // Inline assembly constants must be integers.
+                            Some(fcx.next_int_var())
+                        }
+                        hir::InlineAsmOperand::SymFn { anon_const } if anon_const.hir_id == id => {
+                            Some(fcx.next_ty_var(TypeVariableOrigin {
+                                kind: TypeVariableOriginKind::MiscVariable,
+                                span,
+                            }))
                         }
-                        _ => fallback(),
-                    },
-                    _ => fallback(),
-                });
+                        _ => None,
+                    })
+                }
+                _ => None,
+            }
+        } else {
+            None
+        };
+        let expected_type = expected_type.unwrap_or_else(fallback);
 
-            let expected_type = fcx.normalize(body.value.span, expected_type);
-            fcx.require_type_is_sized(expected_type, body.value.span, traits::ConstSized);
+        let expected_type = fcx.normalize(body.value.span, expected_type);
+        fcx.require_type_is_sized(expected_type, body.value.span, traits::ConstSized);
 
-            // Gather locals in statics (because of block expressions).
-            GatherLocalsVisitor::new(&fcx).visit_body(body);
+        // Gather locals in statics (because of block expressions).
+        GatherLocalsVisitor::new(&fcx).visit_body(body);
 
-            fcx.check_expr_coercable_to_type(&body.value, expected_type, None);
+        fcx.check_expr_coercible_to_type(&body.value, expected_type, None);
 
-            fcx.write_ty(id, expected_type);
-        };
+        fcx.write_ty(id, expected_type);
+    };
 
-        fcx.type_inference_fallback();
-
-        // Even though coercion casts provide type hints, we check casts after fallback for
-        // backwards compatibility. This makes fallback a stronger type hint than a cast coercion.
-        fcx.check_casts();
-        fcx.select_obligations_where_possible(|_| {});
-
-        // Closure and generator analysis may run after fallback
-        // because they don't constrain other type variables.
-        // Closure analysis only runs on closures. Therefore they only need to fulfill non-const predicates (as of now)
-        let prev_constness = fcx.param_env.constness();
-        fcx.param_env = fcx.param_env.without_const();
-        fcx.closure_analyze(body);
-        fcx.param_env = fcx.param_env.with_constness(prev_constness);
-        assert!(fcx.deferred_call_resolutions.borrow().is_empty());
-        // Before the generator analysis, temporary scopes shall be marked to provide more
-        // precise information on types to be captured.
-        fcx.resolve_rvalue_scopes(def_id.to_def_id());
-
-        for (ty, span, code) in fcx.deferred_sized_obligations.borrow_mut().drain(..) {
-            let ty = fcx.normalize(span, ty);
-            fcx.require_type_is_sized(ty, span, code);
-        }
+    fcx.type_inference_fallback();
+
+    // Even though coercion casts provide type hints, we check casts after fallback for
+    // backwards compatibility. This makes fallback a stronger type hint than a cast coercion.
+    fcx.check_casts();
+    fcx.select_obligations_where_possible(|_| {});
+
+    // Closure and generator analysis may run after fallback
+    // because they don't constrain other type variables.
+    // Closure analysis only runs on closures. Therefore they only need to fulfill non-const predicates (as of now)
+    let prev_constness = fcx.param_env.constness();
+    fcx.param_env = fcx.param_env.without_const();
+    fcx.closure_analyze(body);
+    fcx.param_env = fcx.param_env.with_constness(prev_constness);
+    assert!(fcx.deferred_call_resolutions.borrow().is_empty());
+    // Before the generator analysis, temporary scopes shall be marked to provide more
+    // precise information on types to be captured.
+    fcx.resolve_rvalue_scopes(def_id.to_def_id());
+
+    for (ty, span, code) in fcx.deferred_sized_obligations.borrow_mut().drain(..) {
+        let ty = fcx.normalize(span, ty);
+        fcx.require_type_is_sized(ty, span, code);
+    }
 
-        fcx.select_obligations_where_possible(|_| {});
+    fcx.select_obligations_where_possible(|_| {});
 
-        debug!(pending_obligations = ?fcx.fulfillment_cx.borrow().pending_obligations());
+    debug!(pending_obligations = ?fcx.fulfillment_cx.borrow().pending_obligations());
 
-        // This must be the last thing before `report_ambiguity_errors`.
-        fcx.resolve_generator_interiors(def_id.to_def_id());
+    // This must be the last thing before `report_ambiguity_errors`.
+    fcx.resolve_generator_interiors(def_id.to_def_id());
 
-        debug!(pending_obligations = ?fcx.fulfillment_cx.borrow().pending_obligations());
+    debug!(pending_obligations = ?fcx.fulfillment_cx.borrow().pending_obligations());
 
-        if let None = fcx.infcx.tainted_by_errors() {
-            fcx.report_ambiguity_errors();
-        }
+    if let None = fcx.infcx.tainted_by_errors() {
+        fcx.report_ambiguity_errors();
+    }
 
-        if let None = fcx.infcx.tainted_by_errors() {
-            fcx.check_transmutes();
-        }
+    if let None = fcx.infcx.tainted_by_errors() {
+        fcx.check_transmutes();
+    }
 
-        fcx.check_asms();
+    fcx.check_asms();
 
-        fcx.infcx.skip_region_resolution();
+    fcx.infcx.skip_region_resolution();
 
-        fcx.resolve_type_vars_in_body(body)
-    });
+    let typeck_results = fcx.resolve_type_vars_in_body(body);
 
     // Consistency check our TypeckResults instance can hold all ItemLocalIds
     // it will need to hold.
diff --git a/compiler/rustc_hir_typeck/src/mem_categorization.rs b/compiler/rustc_hir_typeck/src/mem_categorization.rs
index 4d3969d28..6c861b593 100644
--- a/compiler/rustc_hir_typeck/src/mem_categorization.rs
+++ b/compiler/rustc_hir_typeck/src/mem_categorization.rs
@@ -59,10 +59,9 @@ use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::pat_util::EnumerateAndAdjustIterator;
 use rustc_hir::PatKind;
-use rustc_index::vec::Idx;
 use rustc_infer::infer::InferCtxt;
 use rustc_span::Span;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT};
 use rustc_trait_selection::infer::InferCtxtExt;
 
 pub(crate) trait HirNode {
@@ -120,8 +119,8 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
         self.infcx.tcx
     }
 
-    pub(crate) fn type_is_copy_modulo_regions(&self, ty: Ty<'tcx>, span: Span) -> bool {
-        self.infcx.type_is_copy_modulo_regions(self.param_env, ty, span)
+    pub(crate) fn type_is_copy_modulo_regions(&self, ty: Ty<'tcx>) -> bool {
+        self.infcx.type_is_copy_modulo_regions(self.param_env, ty)
     }
 
     fn resolve_vars_if_possible<T>(&self, value: T) -> T
@@ -331,7 +330,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
                     expr,
                     base,
                     expr_ty,
-                    ProjectionKind::Field(field_idx as u32, VariantIdx::new(0)),
+                    ProjectionKind::Field(field_idx, FIRST_VARIANT),
                 ))
             }
 
@@ -382,7 +381,6 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
             | hir::ExprKind::Struct(..)
             | hir::ExprKind::Repeat(..)
             | hir::ExprKind::InlineAsm(..)
-            | hir::ExprKind::Box(..)
             | hir::ExprKind::Err(_) => Ok(self.cat_rvalue(expr.hir_id, expr.span, expr_ty)),
         }
     }
@@ -562,7 +560,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
             | Res::SelfTyParam { .. }
             | Res::SelfTyAlias { .. } => {
                 // Structs and Unions have only have one variant.
-                Ok(VariantIdx::new(0))
+                Ok(FIRST_VARIANT)
             }
             _ => bug!("expected ADT path, found={:?}", res),
         }
@@ -676,7 +674,8 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
 
                 for (i, subpat) in subpats.iter().enumerate_and_adjust(total_fields, dots_pos) {
                     let subpat_ty = self.pat_ty_adjusted(subpat)?;
-                    let projection_kind = ProjectionKind::Field(i as u32, VariantIdx::new(0));
+                    let projection_kind =
+                        ProjectionKind::Field(FieldIdx::from_usize(i), FIRST_VARIANT);
                     let sub_place =
                         self.cat_projection(pat, place_with_id.clone(), subpat_ty, projection_kind);
                     self.cat_pattern_(sub_place, subpat, op)?;
@@ -691,7 +690,8 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
 
                 for (i, subpat) in subpats.iter().enumerate_and_adjust(total_fields, dots_pos) {
                     let subpat_ty = self.pat_ty_adjusted(subpat)?;
-                    let projection_kind = ProjectionKind::Field(i as u32, variant_index);
+                    let projection_kind =
+                        ProjectionKind::Field(FieldIdx::from_usize(i), variant_index);
                     let sub_place =
                         self.cat_projection(pat, place_with_id.clone(), subpat_ty, projection_kind);
                     self.cat_pattern_(sub_place, subpat, op)?;
@@ -716,7 +716,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
                         pat,
                         place_with_id.clone(),
                         field_ty,
-                        ProjectionKind::Field(field_index as u32, variant_index),
+                        ProjectionKind::Field(field_index, variant_index),
                     );
                     self.cat_pattern_(field_place, fp.pat, op)?;
                 }
diff --git a/compiler/rustc_hir_typeck/src/method/confirm.rs b/compiler/rustc_hir_typeck/src/method/confirm.rs
index 169f128e0..9155a3d8d 100644
--- a/compiler/rustc_hir_typeck/src/method/confirm.rs
+++ b/compiler/rustc_hir_typeck/src/method/confirm.rs
@@ -8,7 +8,7 @@ use rustc_hir_analysis::astconv::generics::{
     check_generic_arg_count_for_call, create_substs_for_generic_args,
 };
 use rustc_hir_analysis::astconv::{AstConv, CreateSubstsForGenericArgsCtxt, IsMethodCall};
-use rustc_infer::infer::{self, InferOk};
+use rustc_infer::infer::{self, DefineOpaqueTypes, InferOk};
 use rustc_middle::traits::{ObligationCauseCode, UnifyReceiverContext};
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, PointerCast};
 use rustc_middle::ty::adjustment::{AllowTwoPhase, AutoBorrow, AutoBorrowMutability};
@@ -478,7 +478,7 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
                 substs,
             })),
         );
-        match self.at(&cause, self.param_env).sup(method_self_ty, self_ty) {
+        match self.at(&cause, self.param_env).sup(DefineOpaqueTypes::No, method_self_ty, self_ty) {
             Ok(InferOk { obligations, value: () }) => {
                 self.register_predicates(obligations);
             }
@@ -574,19 +574,15 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
     ) -> Option<Span> {
         let sized_def_id = self.tcx.lang_items().sized_trait()?;
 
-        traits::elaborate_predicates(self.tcx, predicates.predicates.iter().copied())
+        traits::elaborate(self.tcx, predicates.predicates.iter().copied())
             // We don't care about regions here.
-            .filter_map(|obligation| match obligation.predicate.kind().skip_binder() {
+            .filter_map(|pred| match pred.kind().skip_binder() {
                 ty::PredicateKind::Clause(ty::Clause::Trait(trait_pred))
                     if trait_pred.def_id() == sized_def_id =>
                 {
                     let span = predicates
                         .iter()
-                        .find_map(
-                            |(p, span)| {
-                                if p == obligation.predicate { Some(span) } else { None }
-                            },
-                        )
+                        .find_map(|(p, span)| if p == pred { Some(span) } else { None })
                         .unwrap_or(rustc_span::DUMMY_SP);
                     Some((trait_pred, span))
                 }
diff --git a/compiler/rustc_hir_typeck/src/method/probe.rs b/compiler/rustc_hir_typeck/src/method/probe.rs
index 3bef5cfcd..4fd778910 100644
--- a/compiler/rustc_hir_typeck/src/method/probe.rs
+++ b/compiler/rustc_hir_typeck/src/method/probe.rs
@@ -13,6 +13,7 @@ use rustc_hir_analysis::astconv::InferCtxtExt as _;
 use rustc_hir_analysis::autoderef::{self, Autoderef};
 use rustc_infer::infer::canonical::OriginalQueryValues;
 use rustc_infer::infer::canonical::{Canonical, QueryResponse};
+use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::infer::{self, InferOk, TyCtxtInferExt};
 use rustc_middle::middle::stability;
 use rustc_middle::ty::fast_reject::{simplify_type, TreatParams};
@@ -699,7 +700,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     }
 
     fn assemble_inherent_candidates_for_incoherent_ty(&mut self, self_ty: Ty<'tcx>) {
-        let Some(simp) = simplify_type(self.tcx, self_ty, TreatParams::AsInfer) else {
+        let Some(simp) = simplify_type(self.tcx, self_ty, TreatParams::AsCandidateKey) else {
             bug!("unexpected incoherent type: {:?}", self_ty)
         };
         for &impl_def_id in self.tcx.incoherent_impls(simp) {
@@ -792,6 +793,14 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         // a `&self` method will wind up with an argument type like `&dyn Trait`.
         let trait_ref = principal.with_self_ty(self.tcx, self_ty);
         self.elaborate_bounds(iter::once(trait_ref), |this, new_trait_ref, item| {
+            if new_trait_ref.has_non_region_late_bound() {
+                this.tcx.sess.delay_span_bug(
+                    this.span,
+                    "tried to select method from HRTB with non-lifetime bound vars",
+                );
+                return;
+            }
+
             let new_trait_ref = this.erase_late_bound_regions(new_trait_ref);
 
             let (xform_self_ty, xform_ret_ty) =
@@ -836,24 +845,21 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 | ty::PredicateKind::ConstEvaluatable(..)
                 | ty::PredicateKind::ConstEquate(..)
                 | ty::PredicateKind::Ambiguous
-                | ty::PredicateKind::AliasEq(..)
+                | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::TypeWellFormedFromEnv(..) => None,
             }
         });
 
         self.elaborate_bounds(bounds, |this, poly_trait_ref, item| {
-            let trait_ref = this.erase_late_bound_regions(poly_trait_ref);
+            let trait_ref = this.instantiate_binder_with_fresh_vars(
+                this.span,
+                infer::LateBoundRegionConversionTime::FnCall,
+                poly_trait_ref,
+            );
 
             let (xform_self_ty, xform_ret_ty) =
                 this.xform_self_ty(item, trait_ref.self_ty(), trait_ref.substs);
 
-            // Because this trait derives from a where-clause, it
-            // should not contain any inference variables or other
-            // artifacts. This means it is safe to put into the
-            // `WhereClauseCandidate` and (eventually) into the
-            // `WhereClausePick`.
-            assert!(!trait_ref.substs.needs_infer());
-
             this.push_candidate(
                 Candidate {
                     xform_self_ty,
@@ -929,7 +935,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 if let Some(self_ty) = self_ty {
                     if self
                         .at(&ObligationCause::dummy(), self.param_env)
-                        .sup(fty.inputs()[0], self_ty)
+                        .sup(DefineOpaqueTypes::No, fty.inputs()[0], self_ty)
                         .is_err()
                     {
                         return false;
@@ -963,7 +969,11 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                             bound_trait_ref.def_id(),
                         ));
                     } else {
-                        let new_trait_ref = self.erase_late_bound_regions(bound_trait_ref);
+                        let new_trait_ref = self.instantiate_binder_with_fresh_vars(
+                            self.span,
+                            infer::LateBoundRegionConversionTime::FnCall,
+                            bound_trait_ref,
+                        );
 
                         let (xform_self_ty, xform_ret_ty) =
                             self.xform_self_ty(item, new_trait_ref.self_ty(), new_trait_ref.substs);
@@ -1026,12 +1036,21 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                     true
                 }
             })
+            // ensure that we don't suggest unstable methods
+            .filter(|candidate| {
+                // note that `DUMMY_SP` is ok here because it is only used for
+                // suggestions and macro stuff which isn't applicable here.
+                !matches!(
+                    self.tcx.eval_stability(candidate.item.def_id, None, DUMMY_SP, None),
+                    stability::EvalResult::Deny { .. }
+                )
+            })
             .map(|candidate| candidate.item.ident(self.tcx))
             .filter(|&name| set.insert(name))
             .collect();
 
         // Sort them by the name so we have a stable result.
-        names.sort_by(|a, b| a.as_str().partial_cmp(b.as_str()).unwrap());
+        names.sort_by(|a, b| a.as_str().cmp(b.as_str()));
         names
     }
 
@@ -1338,6 +1357,7 @@ impl<'tcx> Pick<'tcx> {
                     container: _,
                     trait_item_def_id: _,
                     fn_has_self_parameter: _,
+                    opt_rpitit_info: _,
                 },
             kind: _,
             import_ids: _,
@@ -1434,9 +1454,11 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 CandidateSource::Trait(candidate.item.container_id(self.tcx))
             }
             TraitCandidate(trait_ref) => self.probe(|_| {
-                let _ = self
-                    .at(&ObligationCause::dummy(), self.param_env)
-                    .sup(candidate.xform_self_ty, self_ty);
+                let _ = self.at(&ObligationCause::dummy(), self.param_env).sup(
+                    DefineOpaqueTypes::No,
+                    candidate.xform_self_ty,
+                    self_ty,
+                );
                 match self.select_trait_candidate(trait_ref) {
                     Ok(Some(traits::ImplSource::UserDefined(ref impl_data))) => {
                         // If only a single impl matches, make the error message point
@@ -1463,10 +1485,11 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
 
         self.probe(|_| {
             // First check that the self type can be related.
-            let sub_obligations = match self
-                .at(&ObligationCause::dummy(), self.param_env)
-                .sup(probe.xform_self_ty, self_ty)
-            {
+            let sub_obligations = match self.at(&ObligationCause::dummy(), self.param_env).sup(
+                DefineOpaqueTypes::No,
+                probe.xform_self_ty,
+                self_ty,
+            ) {
                 Ok(InferOk { obligations, value: () }) => obligations,
                 Err(err) => {
                     debug!("--> cannot relate self-types {:?}", err);
@@ -1508,23 +1531,18 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
 
                     // Convert the bounds into obligations.
                     let impl_obligations = traits::predicates_for_generics(
-                        |_idx, span| {
-                            let misc = traits::ObligationCause::misc(span, self.body_id);
-                            let parent_trait_pred = ty::Binder::dummy(ty::TraitPredicate {
-                                trait_ref: ty::TraitRef::from_method(self.tcx, impl_def_id, substs),
-                                constness: ty::BoundConstness::NotConst,
-                                polarity: ty::ImplPolarity::Positive,
-                            });
-                            misc.derived_cause(parent_trait_pred, |derived| {
-                                traits::ImplDerivedObligation(Box::new(
-                                    traits::ImplDerivedObligationCause {
-                                        derived,
-                                        impl_or_alias_def_id: impl_def_id,
-                                        impl_def_predicate_index: None,
-                                        span,
-                                    },
-                                ))
-                            })
+                        |idx, span| {
+                            let code = if span.is_dummy() {
+                                traits::ExprItemObligation(impl_def_id, self.scope_expr_id, idx)
+                            } else {
+                                traits::ExprBindingObligation(
+                                    impl_def_id,
+                                    span,
+                                    self.scope_expr_id,
+                                    idx,
+                                )
+                            };
+                            ObligationCause::new(self.span, self.body_id, code)
                         },
                         self.param_env,
                         impl_bounds,
@@ -1541,8 +1559,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                         if !self.predicate_may_hold(&o) {
                             result = ProbeResult::NoMatch;
                             let parent_o = o.clone();
-                            let implied_obligations =
-                                traits::elaborate_obligations(self.tcx, vec![o]);
+                            let implied_obligations = traits::elaborate(self.tcx, vec![o]);
                             for o in implied_obligations {
                                 let parent = if o == parent_o {
                                     None
@@ -1681,7 +1698,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 if let ProbeResult::Match = result
                     && self
                     .at(&ObligationCause::dummy(), self.param_env)
-                    .sup(return_ty, xform_ret_ty)
+                    .sup(DefineOpaqueTypes::No, return_ty, xform_ret_ty)
                     .is_err()
                 {
                     result = ProbeResult::BadReturnType;
@@ -1742,7 +1759,6 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     fn probe_for_similar_candidate(&mut self) -> Result<Option<ty::AssocItem>, MethodError<'tcx>> {
         debug!("probing for method names similar to {:?}", self.method_name);
 
-        let steps = self.steps.clone();
         self.probe(|_| {
             let mut pcx = ProbeContext::new(
                 self.fcx,
@@ -1750,8 +1766,8 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 self.mode,
                 self.method_name,
                 self.return_type,
-                &self.orig_steps_var_values,
-                steps,
+                self.orig_steps_var_values,
+                self.steps,
                 self.scope_expr_id,
             );
             pcx.allow_similar_names = true;
diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs
index 60d56263d..900a6fa0d 100644
--- a/compiler/rustc_hir_typeck/src/method/suggest.rs
+++ b/compiler/rustc_hir_typeck/src/method/suggest.rs
@@ -27,7 +27,7 @@ use rustc_middle::traits::util::supertraits;
 use rustc_middle::ty::fast_reject::DeepRejectCtxt;
 use rustc_middle::ty::fast_reject::{simplify_type, TreatParams};
 use rustc_middle::ty::print::{with_crate_prefix, with_forced_trimmed_paths};
-use rustc_middle::ty::{self, DefIdTree, GenericArgKind, Ty, TyCtxt, TypeVisitableExt};
+use rustc_middle::ty::{self, GenericArgKind, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::ty::{IsSuggestable, ToPolyTraitRef};
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Symbol;
@@ -42,7 +42,7 @@ use rustc_trait_selection::traits::{
 use super::probe::{AutorefOrPtrAdjustment, IsSuggestion, Mode, ProbeScope};
 use super::{CandidateSource, MethodError, NoMatchData};
 use rustc_hir::intravisit::Visitor;
-use std::cmp::Ordering;
+use std::cmp::{self, Ordering};
 use std::iter;
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
@@ -245,6 +245,28 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         None
     }
 
+    fn suggest_missing_writer(
+        &self,
+        rcvr_ty: Ty<'tcx>,
+        args: (&'tcx hir::Expr<'tcx>, &'tcx [hir::Expr<'tcx>]),
+    ) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
+        let (ty_str, _ty_file) = self.tcx.short_ty_string(rcvr_ty);
+        let mut err =
+            struct_span_err!(self.tcx.sess, args.0.span, E0599, "cannot write into `{}`", ty_str);
+        err.span_note(
+            args.0.span,
+            "must implement `io::Write`, `fmt::Write`, or have a `write_fmt` method",
+        );
+        if let ExprKind::Lit(_) = args.0.kind {
+            err.span_help(
+                args.0.span.shrink_to_lo(),
+                "a writer is needed before this format string",
+            );
+        };
+
+        err
+    }
+
     pub fn report_no_match_method_error(
         &self,
         mut span: Span,
@@ -278,7 +300,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         };
 
         // We could pass the file for long types into these two, but it isn't strictly necessary
-        // given how targetted they are.
+        // given how targeted they are.
         if self.suggest_wrapping_range_with_parens(
             tcx,
             rcvr_ty,
@@ -323,16 +345,26 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         }
 
-        let mut err = struct_span_err!(
-            tcx.sess,
-            span,
-            E0599,
-            "no {} named `{}` found for {} `{}` in the current scope",
-            item_kind,
-            item_name,
-            rcvr_ty.prefix_string(self.tcx),
-            ty_str_reported,
-        );
+        let is_write = sugg_span.ctxt().outer_expn_data().macro_def_id.map_or(false, |def_id| {
+            tcx.is_diagnostic_item(sym::write_macro, def_id)
+                || tcx.is_diagnostic_item(sym::writeln_macro, def_id)
+        }) && item_name.name == Symbol::intern("write_fmt");
+        let mut err = if is_write
+            && let Some(args) = args
+        {
+            self.suggest_missing_writer(rcvr_ty, args)
+        } else {
+            struct_span_err!(
+                tcx.sess,
+                span,
+                E0599,
+                "no {} named `{}` found for {} `{}` in the current scope",
+                item_kind,
+                item_name,
+                rcvr_ty.prefix_string(self.tcx),
+                ty_str_reported,
+            )
+        };
         if tcx.sess.source_map().is_multiline(sugg_span) {
             err.span_label(sugg_span.with_hi(span.lo()), "");
         }
@@ -348,6 +380,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             err.downgrade_to_delayed_bug();
         }
 
+        if tcx.ty_is_opaque_future(rcvr_ty) && item_name.name == sym::poll {
+            err.help(&format!(
+                "method `poll` found on `Pin<&mut {ty_str}>`, \
+                see documentation for `std::pin::Pin`"
+            ));
+            err.help("self type must be pinned to call `Future::poll`, \
+                see https://rust-lang.github.io/async-book/04_pinning/01_chapter.html#pinning-in-practice"
+            );
+        }
+
         if let Mode::MethodCall = mode && let SelfSource::MethodCall(cal) = source {
             self.suggest_await_before_method(
                 &mut err, item_name, rcvr_ty, cal, span, expected.only_has_type(self),
@@ -405,6 +447,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 );
                 probe.is_ok()
             });
+
+            self.note_internal_mutation_in_method(
+                &mut err,
+                rcvr_expr,
+                expected.to_option(&self),
+                rcvr_ty,
+            );
         }
 
         let mut custom_span_label = false;
@@ -612,19 +661,26 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Find all the requirements that come from a local `impl` block.
             let mut skip_list: FxHashSet<_> = Default::default();
             let mut spanned_predicates = FxHashMap::default();
-            for (p, parent_p, impl_def_id, cause) in unsatisfied_predicates
-                .iter()
-                .filter_map(|(p, parent, c)| c.as_ref().map(|c| (p, parent, c)))
-                .filter_map(|(p, parent, c)| match c.code() {
-                    ObligationCauseCode::ImplDerivedObligation(data)
-                        if matches!(p.kind().skip_binder(), ty::PredicateKind::Clause(_)) =>
-                    {
-                        Some((p, parent, data.impl_or_alias_def_id, data))
+            for (p, parent_p, cause) in unsatisfied_predicates {
+                // Extract the predicate span and parent def id of the cause,
+                // if we have one.
+                let (item_def_id, cause_span) = match cause.as_ref().map(|cause| cause.code()) {
+                    Some(ObligationCauseCode::ImplDerivedObligation(data)) => {
+                        (data.impl_or_alias_def_id, data.span)
                     }
-                    _ => None,
-                })
-            {
-                match self.tcx.hir().get_if_local(impl_def_id) {
+                    Some(
+                        ObligationCauseCode::ExprBindingObligation(def_id, span, _, _)
+                        | ObligationCauseCode::BindingObligation(def_id, span),
+                    ) => (*def_id, *span),
+                    _ => continue,
+                };
+
+                // Don't point out the span of `WellFormed` predicates.
+                if !matches!(p.kind().skip_binder(), ty::PredicateKind::Clause(_)) {
+                    continue;
+                };
+
+                match self.tcx.hir().get_if_local(item_def_id) {
                     // Unmet obligation comes from a `derive` macro, point at it once to
                     // avoid multiple span labels pointing at the same place.
                     Some(Node::Item(hir::Item {
@@ -669,7 +725,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 }
                             });
                         for param in generics.params {
-                            if param.span == cause.span && sized_pred {
+                            if param.span == cause_span && sized_pred {
                                 let (sp, sugg) = match param.colon_span {
                                     Some(sp) => (sp.shrink_to_hi(), " ?Sized +"),
                                     None => (param.span.shrink_to_hi(), ": ?Sized"),
@@ -692,9 +748,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             (FxHashSet::default(), FxHashSet::default(), Vec::new())
                         });
                         entry.2.push(p);
-                        if cause.span != *item_span {
-                            entry.0.insert(cause.span);
-                            entry.1.insert((cause.span, "unsatisfied trait bound introduced here"));
+                        if cause_span != *item_span {
+                            entry.0.insert(cause_span);
+                            entry.1.insert((cause_span, "unsatisfied trait bound introduced here"));
                         } else {
                             if let Some(trait_ref) = of_trait {
                                 entry.0.insert(trait_ref.path.span);
@@ -726,9 +782,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         let entry = entry.or_insert_with(|| {
                             (FxHashSet::default(), FxHashSet::default(), Vec::new())
                         });
-                        entry.0.insert(cause.span);
+                        entry.0.insert(cause_span);
                         entry.1.insert((ident.span, ""));
-                        entry.1.insert((cause.span, "unsatisfied trait bound introduced here"));
+                        entry.1.insert((cause_span, "unsatisfied trait bound introduced here"));
                         entry.2.push(p);
                     }
                     Some(node) => unreachable!("encountered `{node:?}`"),
@@ -1164,7 +1220,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 .inputs()
                                 .skip_binder()
                                 .get(0)
-                                .filter(|ty| ty.is_region_ptr() && !rcvr_ty.is_region_ptr())
+                                .filter(|ty| ty.is_ref() && !rcvr_ty.is_ref())
                                 .copied()
                                 .unwrap_or(rcvr_ty),
                         };
@@ -1247,7 +1303,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let target_ty = self
                 .autoderef(sugg_span, rcvr_ty)
                 .find(|(rcvr_ty, _)| {
-                    DeepRejectCtxt { treat_obligation_params: TreatParams::AsInfer }
+                    DeepRejectCtxt { treat_obligation_params: TreatParams::AsCandidateKey }
                         .types_may_unify(*rcvr_ty, impl_ty)
                 })
                 .map_or(impl_ty, |(ty, _)| ty)
@@ -1506,7 +1562,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             .into_iter()
             .any(|info| self.associated_value(info.def_id, item_name).is_some());
         let found_assoc = |ty: Ty<'tcx>| {
-            simplify_type(tcx, ty, TreatParams::AsInfer)
+            simplify_type(tcx, ty, TreatParams::AsCandidateKey)
                 .and_then(|simp| {
                     tcx.incoherent_impls(simp)
                         .iter()
@@ -1766,7 +1822,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     .variants()
                     .iter()
                     .flat_map(|variant| {
-                        let [field] = &variant.fields[..] else { return None; };
+                        let [field] = &variant.fields.raw[..] else { return None; };
                         let field_ty = field.ty(tcx, substs);
 
                         // Skip `_`, since that'll just lead to ambiguity.
@@ -2517,7 +2573,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         if !candidates.is_empty() {
             // Sort from most relevant to least relevant.
-            candidates.sort_by(|a, b| a.cmp(b).reverse());
+            candidates.sort_by_key(|&info| cmp::Reverse(info));
             candidates.dedup();
 
             let param_type = match rcvr_ty.kind() {
@@ -2636,7 +2692,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // cases where a positive bound implies a negative impl.
                 (candidates, Vec::new())
             } else if let Some(simp_rcvr_ty) =
-                simplify_type(self.tcx, rcvr_ty, TreatParams::AsPlaceholder)
+                simplify_type(self.tcx, rcvr_ty, TreatParams::ForLookup)
             {
                 let mut potential_candidates = Vec::new();
                 let mut explicitly_negative = Vec::new();
@@ -2651,7 +2707,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         .any(|imp_did| {
                             let imp = self.tcx.impl_trait_ref(imp_did).unwrap().subst_identity();
                             let imp_simp =
-                                simplify_type(self.tcx, imp.self_ty(), TreatParams::AsPlaceholder);
+                                simplify_type(self.tcx, imp.self_ty(), TreatParams::ForLookup);
                             imp_simp.map_or(false, |s| s == simp_rcvr_ty)
                         })
                     {
diff --git a/compiler/rustc_hir_typeck/src/op.rs b/compiler/rustc_hir_typeck/src/op.rs
index 80279ed96..a52c94cb0 100644
--- a/compiler/rustc_hir_typeck/src/op.rs
+++ b/compiler/rustc_hir_typeck/src/op.rs
@@ -12,16 +12,14 @@ use rustc_middle::ty::adjustment::{
     Adjust, Adjustment, AllowTwoPhase, AutoBorrow, AutoBorrowMutability,
 };
 use rustc_middle::ty::print::with_no_trimmed_paths;
-use rustc_middle::ty::{
-    self, DefIdTree, IsSuggestable, Ty, TyCtxt, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
-};
+use rustc_middle::ty::{self, IsSuggestable, Ty, TyCtxt, TypeVisitableExt};
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::Span;
 use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::traits::error_reporting::suggestions::TypeErrCtxtExt as _;
-use rustc_trait_selection::traits::{self, FulfillmentError};
+use rustc_trait_selection::traits::{self, FulfillmentError, ObligationCtxt};
 use rustc_type_ir::sty::TyKind::*;
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
@@ -103,9 +101,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         match BinOpCategory::from(op) {
             BinOpCategory::Shortcircuit => {
                 // && and || are a simple case.
-                self.check_expr_coercable_to_type(lhs_expr, tcx.types.bool, None);
+                self.check_expr_coercible_to_type(lhs_expr, tcx.types.bool, None);
                 let lhs_diverges = self.diverges.get();
-                self.check_expr_coercable_to_type(rhs_expr, tcx.types.bool, None);
+                self.check_expr_coercible_to_type(rhs_expr, tcx.types.bool, None);
 
                 // Depending on the LHS' value, the RHS can never execute.
                 self.diverges.set(lhs_diverges);
@@ -148,10 +146,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         rhs_ty,
                         op,
                     );
-                    self.demand_suptype(expr.span, builtin_return_ty, return_ty);
+                    self.demand_eqtype(expr.span, builtin_return_ty, return_ty);
+                    builtin_return_ty
+                } else {
+                    return_ty
                 }
-
-                return_ty
             }
         }
     }
@@ -254,7 +253,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         );
 
         // see `NB` above
-        let rhs_ty = self.check_expr_coercable_to_type(rhs_expr, rhs_ty_var, Some(lhs_expr));
+        let rhs_ty = self.check_expr_coercible_to_type(rhs_expr, rhs_ty_var, Some(lhs_expr));
         let rhs_ty = self.resolve_vars_with_obligations(rhs_ty);
 
         let return_ty = match result {
@@ -433,7 +432,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     if self.type_is_copy_modulo_regions(
                         self.param_env,
                         *lhs_deref_ty,
-                        lhs_expr.span,
                     ) {
                         suggest_deref_binop(*lhs_deref_ty);
                     }
@@ -775,7 +773,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             (None, Some(trait_did)) => {
                 let (obligation, _) =
                     self.obligation_for_method(cause, trait_did, lhs_ty, Some(input_types));
-                Err(rustc_trait_selection::traits::fully_solve_obligation(self, obligation))
+                // FIXME: This should potentially just add the obligation to the `FnCtxt`
+                let ocx = ObligationCtxt::new(&self.infcx);
+                ocx.register_obligation(obligation);
+                Err(ocx.select_all_or_error())
             }
         }
     }
@@ -962,21 +963,3 @@ fn is_builtin_binop<'tcx>(lhs: Ty<'tcx>, rhs: Ty<'tcx>, op: hir::BinOp) -> bool
         }
     }
 }
-
-struct TypeParamEraser<'a, 'tcx>(&'a FnCtxt<'a, 'tcx>, Span);
-
-impl<'tcx> TypeFolder<TyCtxt<'tcx>> for TypeParamEraser<'_, 'tcx> {
-    fn interner(&self) -> TyCtxt<'tcx> {
-        self.0.tcx
-    }
-
-    fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
-        match ty.kind() {
-            ty::Param(_) => self.0.next_ty_var(TypeVariableOrigin {
-                kind: TypeVariableOriginKind::MiscVariable,
-                span: self.1,
-            }),
-            _ => ty.super_fold_with(self),
-        }
-    }
-}
diff --git a/compiler/rustc_hir_typeck/src/pat.rs b/compiler/rustc_hir_typeck/src/pat.rs
index c36c75e44..af0bd26de 100644
--- a/compiler/rustc_hir_typeck/src/pat.rs
+++ b/compiler/rustc_hir_typeck/src/pat.rs
@@ -19,6 +19,7 @@ use rustc_span::hygiene::DesugaringKind;
 use rustc_span::source_map::{Span, Spanned};
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::{BytePos, DUMMY_SP};
+use rustc_target::abi::FieldIdx;
 use rustc_trait_selection::traits::{ObligationCause, Pattern};
 use ty::VariantDef;
 
@@ -36,6 +37,10 @@ pointers. If you encounter this error you should try to avoid dereferencing the
 You can read more about trait objects in the Trait Objects section of the Reference: \
 https://doc.rust-lang.org/reference/types.html#trait-objects";
 
+fn is_number(text: &str) -> bool {
+    text.chars().all(|c: char| c.is_digit(10))
+}
+
 /// Information about the expected type at the top level of type checking a pattern.
 ///
 /// **NOTE:** This is only for use by diagnostics. Do NOT use for type checking logic!
@@ -238,15 +243,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Note that there are two tests to check that this remains true
         // (`regions-reassign-{match,let}-bound-pointer.rs`).
         //
-        // 2. Things go horribly wrong if we use subtype. The reason for
-        // THIS is a fairly subtle case involving bound regions. See the
-        // `givens` field in `region_constraints`, as well as the test
+        // 2. An outdated issue related to the old HIR borrowck. See the test
         // `regions-relate-bound-regions-on-closures-to-inference-variables.rs`,
-        // for details. Short version is that we must sometimes detect
-        // relationships between specific region variables and regions
-        // bound in a closure signature, and that detection gets thrown
-        // off when we substitute fresh region variables here to enable
-        // subtyping.
     }
 
     /// Compute the new expected type and default binding mode from the old ones
@@ -1098,11 +1096,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 bug!("unexpected pattern type {:?}", pat_ty);
             };
             for (i, subpat) in subpats.iter().enumerate_and_adjust(variant.fields.len(), ddpos) {
-                let field_ty = self.field_ty(subpat.span, &variant.fields[i], substs);
+                let field = &variant.fields[FieldIdx::from_usize(i)];
+                let field_ty = self.field_ty(subpat.span, field, substs);
                 self.check_pat(subpat, field_ty, def_bm, ti);
 
                 self.tcx.check_stability(
-                    variant.fields[i].did,
+                    variant.fields[FieldIdx::from_usize(i)].did,
                     Some(pat.hir_id),
                     subpat.span,
                     None,
@@ -1110,7 +1109,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         } else {
             // Pattern has wrong number of fields.
-            let e = self.e0023(pat.span, res, qpath, subpats, &variant.fields, expected, had_err);
+            let e =
+                self.e0023(pat.span, res, qpath, subpats, &variant.fields.raw, expected, had_err);
             on_error(e);
             return tcx.ty_error(e);
         }
@@ -1340,8 +1340,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Index the struct fields' types.
         let field_map = variant
             .fields
-            .iter()
-            .enumerate()
+            .iter_enumerated()
             .map(|(i, field)| (field.ident(self.tcx).normalize_to_macros_2_0(), (i, field)))
             .collect::<FxHashMap<_, _>>();
 
@@ -1678,7 +1677,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         fields: &'tcx [hir::PatField<'tcx>],
         variant: &ty::VariantDef,
     ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
-        if let (Some(CtorKind::Fn), PatKind::Struct(qpath, ..)) = (variant.ctor_kind(), &pat.kind) {
+        if let (Some(CtorKind::Fn), PatKind::Struct(qpath, pattern_fields, ..)) =
+            (variant.ctor_kind(), &pat.kind)
+        {
+            let is_tuple_struct_match = !pattern_fields.is_empty()
+                && pattern_fields.iter().map(|field| field.ident.name.as_str()).all(is_number);
+            if is_tuple_struct_match {
+                return None;
+            }
+
             let path = rustc_hir_pretty::to_string(rustc_hir_pretty::NO_ANN, |s| {
                 s.print_qpath(qpath, false)
             });
@@ -1900,7 +1907,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 prefix,
                 unmentioned_fields
                     .iter()
-                    .map(|(_, name)| name.to_string())
+                    .map(|(_, name)| {
+                        let field_name = name.to_string();
+                        if is_number(&field_name) {
+                            format!("{}: _", field_name)
+                        } else {
+                            field_name
+                        }
+                    })
                     .collect::<Vec<_>>()
                     .join(", "),
                 if have_inaccessible_fields { ", .." } else { "" },
diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs
index 4a432328c..41a6ad80b 100644
--- a/compiler/rustc_hir_typeck/src/upvar.rs
+++ b/compiler/rustc_hir_typeck/src/upvar.rs
@@ -49,8 +49,7 @@ use rustc_span::{BytePos, Pos, Span, Symbol};
 use rustc_trait_selection::infer::InferCtxtExt;
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_index::vec::Idx;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::FIRST_VARIANT;
 
 use std::iter;
 
@@ -424,7 +423,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // closures. We want to make sure any adjustment that might make us move the place into
                 // the closure gets handled.
                 let (place, capture_kind) =
-                    restrict_precision_for_drop_types(self, place, capture_kind, usage_span);
+                    restrict_precision_for_drop_types(self, place, capture_kind);
 
                 capture_info.capture_kind = capture_kind;
                 (place, capture_info)
@@ -712,10 +711,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     }
                 }
 
-                unreachable!(
-                    "we captured two identical projections: capture1 = {:?}, capture2 = {:?}",
-                    capture1, capture2
+                self.tcx.sess.delay_span_bug(
+                    closure_span,
+                    &format!(
+                        "two identical projections: ({:?}, {:?})",
+                        capture1.place.projections, capture2.place.projections
+                    ),
                 );
+                std::cmp::Ordering::Equal
             });
         }
 
@@ -1402,7 +1405,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         ProjectionKind::Field(..)
                     ))
                 );
-                def.variants().get(VariantIdx::new(0)).unwrap().fields.iter().enumerate().any(
+                def.variants().get(FIRST_VARIANT).unwrap().fields.iter_enumerated().any(
                     |(i, field)| {
                         let paths_using_field = captured_by_move_projs
                             .iter()
@@ -1410,7 +1413,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 if let ProjectionKind::Field(field_idx, _) =
                                     projs.first().unwrap().kind
                                 {
-                                    if (field_idx as usize) == i { Some(&projs[1..]) } else { None }
+                                    if field_idx == i { Some(&projs[1..]) } else { None }
                                 } else {
                                     unreachable!();
                                 }
@@ -1443,7 +1446,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         .filter_map(|projs| {
                             if let ProjectionKind::Field(field_idx, _) = projs.first().unwrap().kind
                             {
-                                if (field_idx as usize) == i { Some(&projs[1..]) } else { None }
+                                if field_idx.index() == i { Some(&projs[1..]) } else { None }
                             } else {
                                 unreachable!();
                             }
@@ -1498,7 +1501,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     fn should_log_capture_analysis(&self, closure_def_id: LocalDefId) -> bool {
-        self.tcx.has_attr(closure_def_id.to_def_id(), sym::rustc_capture_analysis)
+        self.tcx.has_attr(closure_def_id, sym::rustc_capture_analysis)
     }
 
     fn log_capture_analysis_first_pass(
@@ -1822,9 +1825,8 @@ fn restrict_precision_for_drop_types<'a, 'tcx>(
     fcx: &'a FnCtxt<'a, 'tcx>,
     mut place: Place<'tcx>,
     mut curr_mode: ty::UpvarCapture,
-    span: Span,
 ) -> (Place<'tcx>, ty::UpvarCapture) {
-    let is_copy_type = fcx.infcx.type_is_copy_modulo_regions(fcx.param_env, place.ty(), span);
+    let is_copy_type = fcx.infcx.type_is_copy_modulo_regions(fcx.param_env, place.ty());
 
     if let (false, UpvarCapture::ByValue) = (is_copy_type, curr_mode) {
         for i in 0..place.projections.len() {
@@ -1891,14 +1893,13 @@ fn restrict_capture_precision(
 
     for (i, proj) in place.projections.iter().enumerate() {
         match proj.kind {
-            ProjectionKind::Index => {
-                // Arrays are completely captured, so we drop Index projections
+            ProjectionKind::Index | ProjectionKind::Subslice => {
+                // Arrays are completely captured, so we drop Index and Subslice projections
                 truncate_place_to_len_and_update_capture_kind(&mut place, &mut curr_mode, i);
                 return (place, curr_mode);
             }
             ProjectionKind::Deref => {}
             ProjectionKind::Field(..) => {} // ignore
-            ProjectionKind::Subslice => {}  // We never capture this
         }
     }
 
diff --git a/compiler/rustc_hir_typeck/src/writeback.rs b/compiler/rustc_hir_typeck/src/writeback.rs
index 00348f3af..e876fa275 100644
--- a/compiler/rustc_hir_typeck/src/writeback.rs
+++ b/compiler/rustc_hir_typeck/src/writeback.rs
@@ -44,8 +44,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // This attribute causes us to dump some writeback information
         // in the form of errors, which is used for unit tests.
-        let rustc_dump_user_substs =
-            self.tcx.has_attr(item_def_id.to_def_id(), sym::rustc_dump_user_substs);
+        let rustc_dump_user_substs = self.tcx.has_attr(item_def_id, sym::rustc_dump_user_substs);
 
         let mut wbcx = WritebackCx::new(self, body, rustc_dump_user_substs);
         for param in body.params {
@@ -748,7 +747,7 @@ impl<'cx, 'tcx> Resolver<'cx, 'tcx> {
                 .infcx
                 .err_ctxt()
                 .emit_inference_failure_err(
-                    Some(self.body.id()),
+                    self.tcx.hir().body_owner_def_id(self.body.id()),
                     self.span.to_span(self.tcx),
                     p.into(),
                     E0282,
diff --git a/compiler/rustc_incremental/locales/en-US.ftl b/compiler/rustc_incremental/messages.ftl
index 4852ee0d9..b760620e3 100644
--- a/compiler/rustc_incremental/locales/en-US.ftl
+++ b/compiler/rustc_incremental/messages.ftl
@@ -24,7 +24,7 @@ incremental_field_associated_value_expected = associated value expected for `{$n
 incremental_no_field = no field `{$name}`
 
 incremental_assertion_auto =
-    `except` specified DepNodes that can not be affected for \"{$name}\": \"{$e}\"
+    `except` specified DepNodes that can not be affected for "{$name}": "{$e}"
 
 incremental_undefined_clean_dirty_assertions_item =
     clean/dirty auto-assertions not yet defined for Node::Item.node={$kind}
diff --git a/compiler/rustc_incremental/src/lib.rs b/compiler/rustc_incremental/src/lib.rs
index 511e466c2..df958e4a6 100644
--- a/compiler/rustc_incremental/src/lib.rs
+++ b/compiler/rustc_incremental/src/lib.rs
@@ -35,4 +35,4 @@ pub use persist::{build_dep_graph, load_dep_graph, DepGraphFuture};
 use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_macros::fluent_messages;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs
index b839416c9..1d88dfd20 100644
--- a/compiler/rustc_incremental/src/persist/dirty_clean.rs
+++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs
@@ -371,7 +371,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
     fn check_item(&mut self, item_id: LocalDefId) {
         let item_span = self.tcx.def_span(item_id.to_def_id());
         let def_path_hash = self.tcx.def_path_hash(item_id.to_def_id());
-        for attr in self.tcx.get_attrs(item_id.to_def_id(), sym::rustc_clean) {
+        for attr in self.tcx.get_attrs(item_id, sym::rustc_clean) {
             let Some(assertion) = self.assertion_maybe(item_id, attr) else {
                 continue;
             };
diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs
index 73d7e3bec..d6f83838a 100644
--- a/compiler/rustc_incremental/src/persist/fs.rs
+++ b/compiler/rustc_incremental/src/persist/fs.rs
@@ -108,7 +108,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::{base_n, flock};
 use rustc_errors::ErrorGuaranteed;
-use rustc_fs_util::{link_or_copy, LinkOrCopy};
+use rustc_fs_util::{link_or_copy, try_canonicalize, LinkOrCopy};
 use rustc_session::{Session, StableCrateId};
 use rustc_span::Symbol;
 
@@ -223,7 +223,7 @@ pub fn prepare_session_directory(
     // because, on windows, long paths can cause problems;
     // canonicalization inserts this weird prefix that makes windows
     // tolerate long paths.
-    let crate_dir = match crate_dir.canonicalize() {
+    let crate_dir = match try_canonicalize(&crate_dir) {
         Ok(v) => v,
         Err(err) => {
             return Err(sess.emit_err(errors::CanonicalizePath { path: crate_dir, err }));
@@ -297,10 +297,12 @@ pub fn prepare_session_directory(
 /// renaming it to `s-{timestamp}-{svh}` and releasing the file lock.
 /// If there have been compilation errors, however, this function will just
 /// delete the presumably invalid session directory.
-pub fn finalize_session_directory(sess: &Session, svh: Svh) {
+pub fn finalize_session_directory(sess: &Session, svh: Option<Svh>) {
     if sess.opts.incremental.is_none() {
         return;
     }
+    // The svh is always produced when incr. comp. is enabled.
+    let svh = svh.unwrap();
 
     let _timer = sess.timer("incr_comp_finalize_session_directory");
 
@@ -865,7 +867,7 @@ fn all_except_most_recent(
 /// before passing it to std::fs::remove_dir_all(). This will convert the path
 /// into the '\\?\' format, which supports much longer paths.
 fn safe_remove_dir_all(p: &Path) -> io::Result<()> {
-    let canonicalized = match std_fs::canonicalize(p) {
+    let canonicalized = match try_canonicalize(p) {
         Ok(canonicalized) => canonicalized,
         Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
         Err(err) => return Err(err),
@@ -875,7 +877,7 @@ fn safe_remove_dir_all(p: &Path) -> io::Result<()> {
 }
 
 fn safe_remove_file(p: &Path) -> io::Result<()> {
-    let canonicalized = match std_fs::canonicalize(p) {
+    let canonicalized = match try_canonicalize(p) {
         Ok(canonicalized) => canonicalized,
         Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
         Err(err) => return Err(err),
diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs
index cbf169afb..271ab8306 100644
--- a/compiler/rustc_index/src/bit_set.rs
+++ b/compiler/rustc_index/src/bit_set.rs
@@ -1,5 +1,6 @@
 use crate::vec::{Idx, IndexVec};
 use arrayvec::ArrayVec;
+use smallvec::{smallvec, SmallVec};
 use std::fmt;
 use std::iter;
 use std::marker::PhantomData;
@@ -111,7 +112,7 @@ macro_rules! bit_relations_inherent_impls {
 #[derive(Eq, PartialEq, Hash, Decodable, Encodable)]
 pub struct BitSet<T> {
     domain_size: usize,
-    words: Vec<Word>,
+    words: SmallVec<[Word; 2]>,
     marker: PhantomData<T>,
 }
 
@@ -127,14 +128,15 @@ impl<T: Idx> BitSet<T> {
     #[inline]
     pub fn new_empty(domain_size: usize) -> BitSet<T> {
         let num_words = num_words(domain_size);
-        BitSet { domain_size, words: vec![0; num_words], marker: PhantomData }
+        BitSet { domain_size, words: smallvec![0; num_words], marker: PhantomData }
     }
 
     /// Creates a new, filled bitset with a given `domain_size`.
     #[inline]
     pub fn new_filled(domain_size: usize) -> BitSet<T> {
         let num_words = num_words(domain_size);
-        let mut result = BitSet { domain_size, words: vec![!0; num_words], marker: PhantomData };
+        let mut result =
+            BitSet { domain_size, words: smallvec![!0; num_words], marker: PhantomData };
         result.clear_excess_bits();
         result
     }
@@ -1571,7 +1573,7 @@ impl<T: Idx> From<BitSet<T>> for GrowableBitSet<T> {
 pub struct BitMatrix<R: Idx, C: Idx> {
     num_rows: usize,
     num_columns: usize,
-    words: Vec<Word>,
+    words: SmallVec<[Word; 2]>,
     marker: PhantomData<(R, C)>,
 }
 
@@ -1584,7 +1586,7 @@ impl<R: Idx, C: Idx> BitMatrix<R, C> {
         BitMatrix {
             num_rows,
             num_columns,
-            words: vec![0; num_rows * words_per_row],
+            words: smallvec![0; num_rows * words_per_row],
             marker: PhantomData,
         }
     }
@@ -1848,7 +1850,7 @@ impl<R: Idx, C: Idx> SparseBitMatrix<R, C> {
 
     /// Iterates through all the columns set to true in a given row of
     /// the matrix.
-    pub fn iter<'a>(&'a self, row: R) -> impl Iterator<Item = C> + 'a {
+    pub fn iter(&self, row: R) -> impl Iterator<Item = C> + '_ {
         self.row(row).into_iter().flat_map(|r| r.iter())
     }
 
diff --git a/compiler/rustc_index/src/vec.rs b/compiler/rustc_index/src/vec.rs
index 68cdc6d77..ae2f52c51 100644
--- a/compiler/rustc_index/src/vec.rs
+++ b/compiler/rustc_index/src/vec.rs
@@ -1,11 +1,12 @@
 #[cfg(feature = "rustc_serialize")]
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 
+use std::borrow::{Borrow, BorrowMut};
 use std::fmt;
 use std::fmt::Debug;
 use std::hash::Hash;
 use std::marker::PhantomData;
-use std::ops::{Index, IndexMut, RangeBounds};
+use std::ops::{Deref, DerefMut, Index, IndexMut, RangeBounds};
 use std::slice;
 use std::vec;
 
@@ -23,6 +24,7 @@ pub trait Idx: Copy + 'static + Eq + PartialEq + Debug + Hash {
     }
 
     #[inline]
+    #[must_use = "Use `increment_by` if you wanted to update the index in-place"]
     fn plus(self, amount: usize) -> Self {
         Self::new(self.index() + amount)
     }
@@ -51,16 +53,41 @@ impl Idx for u32 {
     }
 }
 
+/// An owned contiguous collection of `T`s, indexed by `I` rather than by `usize`.
+///
+/// While it's possible to use `u32` or `usize` directly for `I`,
+/// you almost certainly want to use a [`newtype_index!`]-generated type instead.
+///
+/// [`newtype_index!`]: ../macro.newtype_index.html
 #[derive(Clone, PartialEq, Eq, Hash)]
+#[repr(transparent)]
 pub struct IndexVec<I: Idx, T> {
     pub raw: Vec<T>,
     _marker: PhantomData<fn(&I)>,
 }
 
+/// A view into contiguous `T`s, indexed by `I` rather than by `usize`.
+///
+/// One common pattern you'll see is code that uses [`IndexVec::from_elem`]
+/// to create the storage needed for a particular "universe" (aka the set of all
+/// the possible keys that need an associated value) then passes that working
+/// area as `&mut IndexSlice<I, T>` to clarify that nothing will be added nor
+/// removed during processing (and, as a bonus, to chase fewer pointers).
+#[derive(PartialEq, Eq, Hash)]
+#[repr(transparent)]
+pub struct IndexSlice<I: Idx, T> {
+    _marker: PhantomData<fn(&I)>,
+    pub raw: [T],
+}
+
 // Whether `IndexVec` is `Send` depends only on the data,
 // not the phantom data.
 unsafe impl<I: Idx, T> Send for IndexVec<I, T> where T: Send {}
 
+// Whether `IndexSlice` is `Send` depends only on the data,
+// not the phantom data.
+unsafe impl<I: Idx, T> Send for IndexSlice<I, T> where T: Send {}
+
 #[cfg(feature = "rustc_serialize")]
 impl<S: Encoder, I: Idx, T: Encodable<S>> Encodable<S> for IndexVec<I, T> {
     fn encode(&self, s: &mut S) {
@@ -81,6 +108,12 @@ impl<I: Idx, T: fmt::Debug> fmt::Debug for IndexVec<I, T> {
     }
 }
 
+impl<I: Idx, T: fmt::Debug> fmt::Debug for IndexSlice<I, T> {
+    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(&self.raw, fmt)
+    }
+}
+
 impl<I: Idx, T> IndexVec<I, T> {
     #[inline]
     pub fn new() -> Self {
@@ -97,8 +130,19 @@ impl<I: Idx, T> IndexVec<I, T> {
         IndexVec { raw: Vec::with_capacity(capacity), _marker: PhantomData }
     }
 
+    /// Creates a new vector with a copy of `elem` for each index in `universe`.
+    ///
+    /// Thus `IndexVec::from_elem(elem, &universe)` is equivalent to
+    /// `IndexVec::<I, _>::from_elem_n(elem, universe.len())`. That can help
+    /// type inference as it ensures that the resulting vector uses the same
+    /// index type as `universe`, rather than something potentially surprising.
+    ///
+    /// For example, if you want to store data for each local in a MIR body,
+    /// using `let mut uses = IndexVec::from_elem(vec![], &body.local_decls);`
+    /// ensures that `uses` is an `IndexVec<Local, _>`, and thus can give
+    /// better error messages later if one accidentally mismatches indices.
     #[inline]
-    pub fn from_elem<S>(elem: T, universe: &IndexVec<I, S>) -> Self
+    pub fn from_elem<S>(elem: T, universe: &IndexSlice<I, S>) -> Self
     where
         T: Clone,
     {
@@ -123,6 +167,16 @@ impl<I: Idx, T> IndexVec<I, T> {
     }
 
     #[inline]
+    pub fn as_slice(&self) -> &IndexSlice<I, T> {
+        IndexSlice::from_raw(&self.raw)
+    }
+
+    #[inline]
+    pub fn as_mut_slice(&mut self) -> &mut IndexSlice<I, T> {
+        IndexSlice::from_raw_mut(&mut self.raw)
+    }
+
+    #[inline]
     pub fn push(&mut self, d: T) -> I {
         let idx = I::new(self.len());
         self.raw.push(d);
@@ -135,32 +189,145 @@ impl<I: Idx, T> IndexVec<I, T> {
     }
 
     #[inline]
-    pub fn len(&self) -> usize {
-        self.raw.len()
+    pub fn into_iter(self) -> vec::IntoIter<T> {
+        self.raw.into_iter()
     }
 
-    /// Gives the next index that will be assigned when `push` is
-    /// called.
     #[inline]
-    pub fn next_index(&self) -> I {
-        I::new(self.len())
+    pub fn into_iter_enumerated(
+        self,
+    ) -> impl DoubleEndedIterator<Item = (I, T)> + ExactSizeIterator {
+        self.raw.into_iter().enumerate().map(|(n, t)| (I::new(n), t))
     }
 
     #[inline]
-    pub fn is_empty(&self) -> bool {
-        self.raw.is_empty()
+    pub fn drain<R: RangeBounds<usize>>(&mut self, range: R) -> impl Iterator<Item = T> + '_ {
+        self.raw.drain(range)
     }
 
     #[inline]
-    pub fn into_iter(self) -> vec::IntoIter<T> {
-        self.raw.into_iter()
+    pub fn drain_enumerated<R: RangeBounds<usize>>(
+        &mut self,
+        range: R,
+    ) -> impl Iterator<Item = (I, T)> + '_ {
+        let begin = match range.start_bound() {
+            std::ops::Bound::Included(i) => *i,
+            std::ops::Bound::Excluded(i) => i.checked_add(1).unwrap(),
+            std::ops::Bound::Unbounded => 0,
+        };
+        self.raw.drain(range).enumerate().map(move |(n, t)| (I::new(begin + n), t))
     }
 
     #[inline]
-    pub fn into_iter_enumerated(
-        self,
-    ) -> impl DoubleEndedIterator<Item = (I, T)> + ExactSizeIterator {
-        self.raw.into_iter().enumerate().map(|(n, t)| (I::new(n), t))
+    pub fn shrink_to_fit(&mut self) {
+        self.raw.shrink_to_fit()
+    }
+
+    #[inline]
+    pub fn truncate(&mut self, a: usize) {
+        self.raw.truncate(a)
+    }
+
+    pub fn convert_index_type<Ix: Idx>(self) -> IndexVec<Ix, T> {
+        IndexVec { raw: self.raw, _marker: PhantomData }
+    }
+
+    /// Grows the index vector so that it contains an entry for
+    /// `elem`; if that is already true, then has no
+    /// effect. Otherwise, inserts new values as needed by invoking
+    /// `fill_value`.
+    #[inline]
+    pub fn ensure_contains_elem(&mut self, elem: I, fill_value: impl FnMut() -> T) {
+        let min_new_len = elem.index() + 1;
+        if self.len() < min_new_len {
+            self.raw.resize_with(min_new_len, fill_value);
+        }
+    }
+
+    #[inline]
+    pub fn resize_to_elem(&mut self, elem: I, fill_value: impl FnMut() -> T) {
+        let min_new_len = elem.index() + 1;
+        self.raw.resize_with(min_new_len, fill_value);
+    }
+}
+
+impl<I: Idx, T> Deref for IndexVec<I, T> {
+    type Target = IndexSlice<I, T>;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        self.as_slice()
+    }
+}
+
+impl<I: Idx, T> DerefMut for IndexVec<I, T> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.as_mut_slice()
+    }
+}
+
+impl<I: Idx, T> Borrow<IndexSlice<I, T>> for IndexVec<I, T> {
+    fn borrow(&self) -> &IndexSlice<I, T> {
+        self
+    }
+}
+
+impl<I: Idx, T> BorrowMut<IndexSlice<I, T>> for IndexVec<I, T> {
+    fn borrow_mut(&mut self) -> &mut IndexSlice<I, T> {
+        self
+    }
+}
+
+impl<I: Idx, T: Clone> ToOwned for IndexSlice<I, T> {
+    type Owned = IndexVec<I, T>;
+
+    fn to_owned(&self) -> IndexVec<I, T> {
+        IndexVec::from_raw(self.raw.to_owned())
+    }
+
+    fn clone_into(&self, target: &mut IndexVec<I, T>) {
+        self.raw.clone_into(&mut target.raw)
+    }
+}
+
+impl<I: Idx, T> IndexSlice<I, T> {
+    #[inline]
+    pub fn empty() -> &'static Self {
+        Default::default()
+    }
+
+    #[inline]
+    pub fn from_raw(raw: &[T]) -> &Self {
+        let ptr: *const [T] = raw;
+        // SAFETY: `IndexSlice` is `repr(transparent)` over a normal slice
+        unsafe { &*(ptr as *const Self) }
+    }
+
+    #[inline]
+    pub fn from_raw_mut(raw: &mut [T]) -> &mut Self {
+        let ptr: *mut [T] = raw;
+        // SAFETY: `IndexSlice` is `repr(transparent)` over a normal slice
+        unsafe { &mut *(ptr as *mut Self) }
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.raw.len()
+    }
+
+    /// Gives the next index that will be assigned when `push` is called.
+    ///
+    /// Manual bounds checks can be done using `idx < slice.next_index()`
+    /// (as opposed to `idx.index() < slice.len()`).
+    #[inline]
+    pub fn next_index(&self) -> I {
+        I::new(self.len())
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.raw.is_empty()
     }
 
     #[inline]
@@ -195,47 +362,16 @@ impl<I: Idx, T> IndexVec<I, T> {
     }
 
     #[inline]
-    pub fn drain<'a, R: RangeBounds<usize>>(
-        &'a mut self,
-        range: R,
-    ) -> impl Iterator<Item = T> + 'a {
-        self.raw.drain(range)
-    }
-
-    #[inline]
-    pub fn drain_enumerated<'a, R: RangeBounds<usize>>(
-        &'a mut self,
-        range: R,
-    ) -> impl Iterator<Item = (I, T)> + 'a {
-        let begin = match range.start_bound() {
-            std::ops::Bound::Included(i) => *i,
-            std::ops::Bound::Excluded(i) => i.checked_add(1).unwrap(),
-            std::ops::Bound::Unbounded => 0,
-        };
-        self.raw.drain(range).enumerate().map(move |(n, t)| (I::new(begin + n), t))
-    }
-
-    #[inline]
-    pub fn last(&self) -> Option<I> {
+    pub fn last_index(&self) -> Option<I> {
         self.len().checked_sub(1).map(I::new)
     }
 
     #[inline]
-    pub fn shrink_to_fit(&mut self) {
-        self.raw.shrink_to_fit()
-    }
-
-    #[inline]
     pub fn swap(&mut self, a: I, b: I) {
         self.raw.swap(a.index(), b.index())
     }
 
     #[inline]
-    pub fn truncate(&mut self, a: usize) {
-        self.raw.truncate(a)
-    }
-
-    #[inline]
     pub fn get(&self, index: I) -> Option<&T> {
         self.raw.get(index.index())
     }
@@ -274,27 +410,35 @@ impl<I: Idx, T> IndexVec<I, T> {
         let ptr = self.raw.as_mut_ptr();
         unsafe { (&mut *ptr.add(ai), &mut *ptr.add(bi), &mut *ptr.add(ci)) }
     }
+}
 
-    pub fn convert_index_type<Ix: Idx>(self) -> IndexVec<Ix, T> {
-        IndexVec { raw: self.raw, _marker: PhantomData }
-    }
-
-    /// Grows the index vector so that it contains an entry for
-    /// `elem`; if that is already true, then has no
-    /// effect. Otherwise, inserts new values as needed by invoking
-    /// `fill_value`.
-    #[inline]
-    pub fn ensure_contains_elem(&mut self, elem: I, fill_value: impl FnMut() -> T) {
-        let min_new_len = elem.index() + 1;
-        if self.len() < min_new_len {
-            self.raw.resize_with(min_new_len, fill_value);
+impl<I: Idx, J: Idx> IndexSlice<I, J> {
+    /// Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`,
+    /// assuming the values in `self` are a permutation of `0..self.len()`.
+    ///
+    /// This is used to go between `memory_index` (source field order to memory order)
+    /// and `inverse_memory_index` (memory order to source field order).
+    /// See also `FieldsShape::Arbitrary::memory_index` for more details.
+    // FIXME(eddyb) build a better abstraction for permutations, if possible.
+    pub fn invert_bijective_mapping(&self) -> IndexVec<J, I> {
+        debug_assert_eq!(
+            self.iter().map(|x| x.index() as u128).sum::<u128>(),
+            (0..self.len() as u128).sum::<u128>(),
+            "The values aren't 0..N in input {self:?}",
+        );
+
+        let mut inverse = IndexVec::from_elem_n(Idx::new(0), self.len());
+        for (i1, &i2) in self.iter_enumerated() {
+            inverse[i2] = i1;
         }
-    }
 
-    #[inline]
-    pub fn resize_to_elem(&mut self, elem: I, fill_value: impl FnMut() -> T) {
-        let min_new_len = elem.index() + 1;
-        self.raw.resize_with(min_new_len, fill_value);
+        debug_assert_eq!(
+            inverse.iter().map(|x| x.index() as u128).sum::<u128>(),
+            (0..inverse.len() as u128).sum::<u128>(),
+            "The values aren't 0..N in result {self:?}",
+        );
+
+        inverse
     }
 }
 
@@ -326,7 +470,7 @@ impl<I: Idx, T: Clone> IndexVec<I, T> {
     }
 }
 
-impl<I: Idx, T: Ord> IndexVec<I, T> {
+impl<I: Idx, T: Ord> IndexSlice<I, T> {
     #[inline]
     pub fn binary_search(&self, value: &T) -> Result<I, I> {
         match self.raw.binary_search(value) {
@@ -336,7 +480,7 @@ impl<I: Idx, T: Ord> IndexVec<I, T> {
     }
 }
 
-impl<I: Idx, T> Index<I> for IndexVec<I, T> {
+impl<I: Idx, T> Index<I> for IndexSlice<I, T> {
     type Output = T;
 
     #[inline]
@@ -345,7 +489,7 @@ impl<I: Idx, T> Index<I> for IndexVec<I, T> {
     }
 }
 
-impl<I: Idx, T> IndexMut<I> for IndexVec<I, T> {
+impl<I: Idx, T> IndexMut<I> for IndexSlice<I, T> {
     #[inline]
     fn index_mut(&mut self, index: I) -> &mut T {
         &mut self.raw[index.index()]
@@ -359,6 +503,20 @@ impl<I: Idx, T> Default for IndexVec<I, T> {
     }
 }
 
+impl<I: Idx, T> Default for &IndexSlice<I, T> {
+    #[inline]
+    fn default() -> Self {
+        IndexSlice::from_raw(Default::default())
+    }
+}
+
+impl<I: Idx, T> Default for &mut IndexSlice<I, T> {
+    #[inline]
+    fn default() -> Self {
+        IndexSlice::from_raw_mut(Default::default())
+    }
+}
+
 impl<I: Idx, T> Extend<T> for IndexVec<I, T> {
     #[inline]
     fn extend<J: IntoIterator<Item = T>>(&mut self, iter: J) {
@@ -388,6 +546,13 @@ impl<I: Idx, T> FromIterator<T> for IndexVec<I, T> {
     }
 }
 
+impl<I: Idx, T, const N: usize> From<[T; N]> for IndexVec<I, T> {
+    #[inline]
+    fn from(array: [T; N]) -> Self {
+        IndexVec::from_raw(array.into())
+    }
+}
+
 impl<I: Idx, T> IntoIterator for IndexVec<I, T> {
     type Item = T;
     type IntoIter = vec::IntoIter<T>;
@@ -418,5 +583,25 @@ impl<'a, I: Idx, T> IntoIterator for &'a mut IndexVec<I, T> {
     }
 }
 
+impl<'a, I: Idx, T> IntoIterator for &'a IndexSlice<I, T> {
+    type Item = &'a T;
+    type IntoIter = slice::Iter<'a, T>;
+
+    #[inline]
+    fn into_iter(self) -> slice::Iter<'a, T> {
+        self.raw.iter()
+    }
+}
+
+impl<'a, I: Idx, T> IntoIterator for &'a mut IndexSlice<I, T> {
+    type Item = &'a mut T;
+    type IntoIter = slice::IterMut<'a, T>;
+
+    #[inline]
+    fn into_iter(self) -> slice::IterMut<'a, T> {
+        self.raw.iter_mut()
+    }
+}
+
 #[cfg(test)]
 mod tests;
diff --git a/compiler/rustc_infer/locales/en-US.ftl b/compiler/rustc_infer/messages.ftl
index 15780898d..c8998ea91 100644
--- a/compiler/rustc_infer/locales/en-US.ftl
+++ b/compiler/rustc_infer/messages.ftl
@@ -30,8 +30,8 @@ infer_source_kind_subdiag_let = {$kind ->
 }{$x_kind ->
     [has_name] , where the {$prefix_kind ->
         *[type] type for {$prefix}
-        [const_with_param] the value of const parameter
-        [const] the value of the constant
+        [const_with_param] value of const parameter
+        [const] value of the constant
     } `{$arg_name}` is specified
     [underscore] , where the placeholders `_` are specified
     *[empty] {""}
@@ -163,7 +163,6 @@ infer_region_explanation = {$pref_kind ->
     [as_defined] the lifetime `{$desc_arg}` as defined here
     [as_defined_anon] the anonymous lifetime as defined here
     [defined_here] the anonymous lifetime defined here
-    [anon_num_here] the anonymous lifetime #{$desc_num_arg} defined here
     [defined_here_reg] the lifetime `{$desc_arg}` as defined here
 }{$suff_kind ->
     *[should_not_happen] [{$suff_kind}]
@@ -174,7 +173,7 @@ infer_region_explanation = {$pref_kind ->
 
 infer_outlives_content = lifetime of reference outlives lifetime of borrowed content...
 infer_outlives_bound = lifetime of the source pointer does not outlive lifetime bound of the object type
-infer_fullfill_req_lifetime = the type `{$ty}` does not fulfill the required lifetime
+infer_fulfill_req_lifetime = the type `{$ty}` does not fulfill the required lifetime
 infer_lf_bound_not_satisfied = lifetime bound not satisfied
 infer_borrowed_too_long = a value of type `{$ty}` is borrowed for too long
 infer_ref_longer_than_data = in type `{$ty}`, reference has a longer lifetime than the data it references
@@ -348,3 +347,47 @@ infer_prlf_known_limitation = this is a known limitation that will be removed in
 
 infer_opaque_captures_lifetime = hidden type for `{$opaque_ty}` captures lifetime that does not appear in bounds
     .label = opaque type defined here
+
+infer_fps_use_ref = consider using a reference
+infer_fps_remove_ref = consider removing the reference
+infer_fps_cast = consider casting to a fn pointer
+infer_fps_items_are_distinct = fn items are distinct from fn pointers
+infer_fps_cast_both = consider casting both fn items to fn pointers using `as {$expected_sig}`
+
+infer_fn_uniq_types = different fn items have unique types, even if their signatures are the same
+infer_fn_consider_casting = consider casting the fn item to a fn pointer: `{$casting}`
+
+infer_sarwa_option = you can convert from `&Option<T>` to `Option<&T>` using `.as_ref()`
+infer_sarwa_result = you can convert from `&Result<T, E>` to `Result<&T, &E>` using `.as_ref()`
+
+infer_suggest_accessing_field = you might have meant to use field `{$name}` whose type is `{$ty}`
+
+infer_sbfrit_change_return_type = you could change the return type to be a boxed trait object
+infer_sbfrit_box_return_expr = if you change the return type to expect trait objects, box the returned expressions
+
+infer_stp_wrap_one = try wrapping the pattern in `{$variant}`
+infer_stp_wrap_many = try wrapping the pattern in a variant of `{$path}`
+
+infer_tuple_trailing_comma = use a trailing comma to create a tuple with one element
+
+infer_oc_method_compat = method not compatible with trait
+infer_oc_type_compat = type not compatible with trait
+infer_oc_const_compat = const not compatible with trait
+infer_oc_try_compat = `?` operator has incompatible types
+infer_oc_match_compat = `match` arms have incompatible types
+infer_oc_if_else_different = `if` and `else` have incompatible types
+infer_oc_no_else = `if` may be missing an `else` clause
+infer_oc_no_diverge = `else` clause of `let...else` does not diverge
+infer_oc_fn_main_correct_type = `main` function has wrong type
+infer_oc_fn_start_correct_type = `#[start]` function has wrong type
+infer_oc_intristic_correct_type = intrinsic has wrong type
+infer_oc_method_correct_type = mismatched `self` parameter type
+infer_oc_closure_selfref = closure/generator type that references itself
+infer_oc_cant_coerce = cannot coerce intrinsics to function pointers
+infer_oc_generic = mismatched types
+
+infer_meant_byte_literal = if you meant to write a byte literal, prefix with `b`
+infer_meant_char_literal = if you meant to write a `char` literal, use single quotes
+infer_meant_str_literal = if you meant to write a `str` literal, use double quotes
+infer_consider_specifying_length = consider specifying the actual array length
+infer_try_cannot_convert = `?` operator cannot convert from `{$found}` to `{$expected}`
diff --git a/compiler/rustc_infer/src/errors/mod.rs b/compiler/rustc_infer/src/errors/mod.rs
index 6bbd3fd3e..65b3dd1a8 100644
--- a/compiler/rustc_infer/src/errors/mod.rs
+++ b/compiler/rustc_infer/src/errors/mod.rs
@@ -53,7 +53,7 @@ pub struct AnnotationRequired<'a> {
 // Copy of `AnnotationRequired` for E0283
 #[derive(Diagnostic)]
 #[diag(infer_type_annotations_needed, code = "E0283")]
-pub struct AmbigousImpl<'a> {
+pub struct AmbiguousImpl<'a> {
     #[primary_span]
     pub span: Span,
     pub source_kind: &'static str,
@@ -184,18 +184,6 @@ pub enum SourceKindMultiSuggestion<'a> {
     },
 }
 
-#[derive(Subdiagnostic)]
-#[suggestion(
-    infer_suggest_add_let_for_letchains,
-    style = "verbose",
-    applicability = "machine-applicable",
-    code = "let "
-)]
-pub(crate) struct SuggAddLetForLetChains {
-    #[primary_span]
-    pub span: Span,
-}
-
 impl<'a> SourceKindMultiSuggestion<'a> {
     pub fn new_fully_qualified(
         span: Span,
@@ -954,8 +942,8 @@ pub struct OutlivesBound<'a> {
 }
 
 #[derive(Diagnostic)]
-#[diag(infer_fullfill_req_lifetime, code = "E0477")]
-pub struct FullfillReqLifetime<'a> {
+#[diag(infer_fulfill_req_lifetime, code = "E0477")]
+pub struct FulfillReqLifetime<'a> {
     #[primary_span]
     pub span: Span,
     pub ty: Ty<'a>,
@@ -1157,3 +1145,380 @@ pub struct OpaqueCapturesLifetime<'tcx> {
     pub opaque_ty_span: Span,
     pub opaque_ty: Ty<'tcx>,
 }
+
+#[derive(Subdiagnostic)]
+pub enum FunctionPointerSuggestion<'a> {
+    #[suggestion(
+        infer_fps_use_ref,
+        code = "&{fn_name}",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
+    UseRef {
+        #[primary_span]
+        span: Span,
+        #[skip_arg]
+        fn_name: String,
+    },
+    #[suggestion(
+        infer_fps_remove_ref,
+        code = "{fn_name}",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
+    RemoveRef {
+        #[primary_span]
+        span: Span,
+        #[skip_arg]
+        fn_name: String,
+    },
+    #[suggestion(
+        infer_fps_cast,
+        code = "&({fn_name} as {sig})",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
+    CastRef {
+        #[primary_span]
+        span: Span,
+        #[skip_arg]
+        fn_name: String,
+        #[skip_arg]
+        sig: Binder<'a, FnSig<'a>>,
+    },
+    #[suggestion(
+        infer_fps_cast,
+        code = "{fn_name} as {sig}",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
+    Cast {
+        #[primary_span]
+        span: Span,
+        #[skip_arg]
+        fn_name: String,
+        #[skip_arg]
+        sig: Binder<'a, FnSig<'a>>,
+    },
+    #[suggestion(
+        infer_fps_cast_both,
+        code = "{fn_name} as {found_sig}",
+        style = "hidden",
+        applicability = "maybe-incorrect"
+    )]
+    CastBoth {
+        #[primary_span]
+        span: Span,
+        #[skip_arg]
+        fn_name: String,
+        #[skip_arg]
+        found_sig: Binder<'a, FnSig<'a>>,
+        expected_sig: Binder<'a, FnSig<'a>>,
+    },
+    #[suggestion(
+        infer_fps_cast_both,
+        code = "&({fn_name} as {found_sig})",
+        style = "hidden",
+        applicability = "maybe-incorrect"
+    )]
+    CastBothRef {
+        #[primary_span]
+        span: Span,
+        #[skip_arg]
+        fn_name: String,
+        #[skip_arg]
+        found_sig: Binder<'a, FnSig<'a>>,
+        expected_sig: Binder<'a, FnSig<'a>>,
+    },
+}
+
+#[derive(Subdiagnostic)]
+#[note(infer_fps_items_are_distinct)]
+pub struct FnItemsAreDistinct;
+
+#[derive(Subdiagnostic)]
+#[note(infer_fn_uniq_types)]
+pub struct FnUniqTypes;
+
+#[derive(Subdiagnostic)]
+#[help(infer_fn_consider_casting)]
+pub struct FnConsiderCasting {
+    pub casting: String,
+}
+
+#[derive(Subdiagnostic)]
+pub enum SuggestAsRefWhereAppropriate<'a> {
+    #[suggestion(
+        infer_sarwa_option,
+        code = "{snippet}.as_ref()",
+        applicability = "machine-applicable"
+    )]
+    Option {
+        #[primary_span]
+        span: Span,
+        snippet: &'a str,
+    },
+    #[suggestion(
+        infer_sarwa_result,
+        code = "{snippet}.as_ref()",
+        applicability = "machine-applicable"
+    )]
+    Result {
+        #[primary_span]
+        span: Span,
+        snippet: &'a str,
+    },
+}
+
+#[derive(Subdiagnostic)]
+pub enum SuggestAccessingField<'a> {
+    #[suggestion(
+        infer_suggest_accessing_field,
+        code = "{snippet}.{name}",
+        applicability = "maybe-incorrect"
+    )]
+    Safe {
+        #[primary_span]
+        span: Span,
+        snippet: String,
+        name: Symbol,
+        ty: Ty<'a>,
+    },
+    #[suggestion(
+        infer_suggest_accessing_field,
+        code = "unsafe {{ {snippet}.{name} }}",
+        applicability = "maybe-incorrect"
+    )]
+    Unsafe {
+        #[primary_span]
+        span: Span,
+        snippet: String,
+        name: Symbol,
+        ty: Ty<'a>,
+    },
+}
+
+#[derive(Subdiagnostic)]
+pub enum SuggestBoxingForReturnImplTrait {
+    #[multipart_suggestion(infer_sbfrit_change_return_type, applicability = "maybe-incorrect")]
+    ChangeReturnType {
+        #[suggestion_part(code = "Box<dyn")]
+        start_sp: Span,
+        #[suggestion_part(code = ">")]
+        end_sp: Span,
+    },
+    #[multipart_suggestion(infer_sbfrit_box_return_expr, applicability = "maybe-incorrect")]
+    BoxReturnExpr {
+        #[suggestion_part(code = "Box::new(")]
+        starts: Vec<Span>,
+        #[suggestion_part(code = ")")]
+        ends: Vec<Span>,
+    },
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(infer_stp_wrap_one, applicability = "maybe-incorrect")]
+pub struct SuggestTuplePatternOne {
+    pub variant: String,
+    #[suggestion_part(code = "{variant}(")]
+    pub span_low: Span,
+    #[suggestion_part(code = ")")]
+    pub span_high: Span,
+}
+
+pub struct SuggestTuplePatternMany {
+    pub path: String,
+    pub cause_span: Span,
+    pub compatible_variants: Vec<String>,
+}
+
+impl AddToDiagnostic for SuggestTuplePatternMany {
+    fn add_to_diagnostic_with<F>(self, diag: &mut rustc_errors::Diagnostic, f: F)
+    where
+        F: Fn(&mut Diagnostic, SubdiagnosticMessage) -> SubdiagnosticMessage,
+    {
+        diag.set_arg("path", self.path);
+        let message = f(diag, crate::fluent_generated::infer_stp_wrap_many.into());
+        diag.multipart_suggestions(
+            message,
+            self.compatible_variants.into_iter().map(|variant| {
+                vec![
+                    (self.cause_span.shrink_to_lo(), format!("{}(", variant)),
+                    (self.cause_span.shrink_to_hi(), ")".to_string()),
+                ]
+            }),
+            rustc_errors::Applicability::MaybeIncorrect,
+        );
+    }
+}
+
+#[derive(Subdiagnostic)]
+pub enum TypeErrorAdditionalDiags {
+    #[suggestion(
+        infer_meant_byte_literal,
+        code = "b'{code}'",
+        applicability = "machine-applicable"
+    )]
+    MeantByteLiteral {
+        #[primary_span]
+        span: Span,
+        code: String,
+    },
+    #[suggestion(
+        infer_meant_char_literal,
+        code = "'{code}'",
+        applicability = "machine-applicable"
+    )]
+    MeantCharLiteral {
+        #[primary_span]
+        span: Span,
+        code: String,
+    },
+    #[suggestion(
+        infer_meant_str_literal,
+        code = "\"{code}\"",
+        applicability = "machine-applicable"
+    )]
+    MeantStrLiteral {
+        #[primary_span]
+        span: Span,
+        code: String,
+    },
+    #[suggestion(
+        infer_consider_specifying_length,
+        code = "{length}",
+        applicability = "maybe-incorrect"
+    )]
+    ConsiderSpecifyingLength {
+        #[primary_span]
+        span: Span,
+        length: u64,
+    },
+    #[note(infer_try_cannot_convert)]
+    TryCannotConvert { found: String, expected: String },
+    #[suggestion(infer_tuple_trailing_comma, code = ",", applicability = "machine-applicable")]
+    TupleOnlyComma {
+        #[primary_span]
+        span: Span,
+    },
+    #[multipart_suggestion(infer_tuple_trailing_comma, applicability = "machine-applicable")]
+    TupleAlsoParentheses {
+        #[suggestion_part(code = "(")]
+        span_low: Span,
+        #[suggestion_part(code = ",)")]
+        span_high: Span,
+    },
+    #[suggestion(
+        infer_suggest_add_let_for_letchains,
+        style = "verbose",
+        applicability = "machine-applicable",
+        code = "let "
+    )]
+    AddLetForLetChains {
+        #[primary_span]
+        span: Span,
+    },
+}
+
+#[derive(Diagnostic)]
+pub enum ObligationCauseFailureCode {
+    #[diag(infer_oc_method_compat, code = "E0308")]
+    MethodCompat {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_type_compat, code = "E0308")]
+    TypeCompat {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_const_compat, code = "E0308")]
+    ConstCompat {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_try_compat, code = "E0308")]
+    TryCompat {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_match_compat, code = "E0308")]
+    MatchCompat {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_if_else_different, code = "E0308")]
+    IfElseDifferent {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_no_else, code = "E0317")]
+    NoElse {
+        #[primary_span]
+        span: Span,
+    },
+    #[diag(infer_oc_no_diverge, code = "E0308")]
+    NoDiverge {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_fn_main_correct_type, code = "E0580")]
+    FnMainCorrectType {
+        #[primary_span]
+        span: Span,
+    },
+    #[diag(infer_oc_fn_start_correct_type, code = "E0308")]
+    FnStartCorrectType {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_intristic_correct_type, code = "E0308")]
+    IntristicCorrectType {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_method_correct_type, code = "E0308")]
+    MethodCorrectType {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_closure_selfref, code = "E0644")]
+    ClosureSelfref {
+        #[primary_span]
+        span: Span,
+    },
+    #[diag(infer_oc_cant_coerce, code = "E0308")]
+    CantCoerce {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(infer_oc_generic, code = "E0308")]
+    Generic {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+}
diff --git a/compiler/rustc_infer/src/errors/note_and_explain.rs b/compiler/rustc_infer/src/errors/note_and_explain.rs
index ef543b1fb..7328241df 100644
--- a/compiler/rustc_infer/src/errors/note_and_explain.rs
+++ b/compiler/rustc_infer/src/errors/note_and_explain.rs
@@ -4,12 +4,10 @@ use rustc_errors::{self, AddToDiagnostic, Diagnostic, IntoDiagnosticArg, Subdiag
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::{symbol::kw, Span};
 
-#[derive(Default)]
 struct DescriptionCtx<'a> {
     span: Option<Span>,
     kind: &'a str,
     arg: String,
-    num_arg: u32,
 }
 
 impl<'a> DescriptionCtx<'a> {
@@ -18,103 +16,74 @@ impl<'a> DescriptionCtx<'a> {
         region: ty::Region<'tcx>,
         alt_span: Option<Span>,
     ) -> Option<Self> {
-        let mut me = DescriptionCtx::default();
-        me.span = alt_span;
-        match *region {
-            ty::ReEarlyBound(_) | ty::ReFree(_) => {
-                return Self::from_early_bound_and_free_regions(tcx, region);
-            }
-            ty::ReStatic => {
-                me.kind = "restatic";
-            }
-
-            ty::RePlaceholder(_) => return None,
-
-            ty::ReError(_) => return None,
-
-            // FIXME(#13998) RePlaceholder should probably print like
-            // ReFree rather than dumping Debug output on the user.
-            //
-            // We shouldn't really be having unification failures with ReVar
-            // and ReLateBound though.
-            ty::ReVar(_) | ty::ReLateBound(..) | ty::ReErased => {
-                me.kind = "revar";
-                me.arg = format!("{:?}", region);
-            }
-        };
-        Some(me)
-    }
-
-    fn from_early_bound_and_free_regions<'tcx>(
-        tcx: TyCtxt<'tcx>,
-        region: ty::Region<'tcx>,
-    ) -> Option<Self> {
-        let mut me = DescriptionCtx::default();
-        let scope = region.free_region_binding_scope(tcx).expect_local();
-        match *region {
+        let (span, kind, arg) = match *region {
             ty::ReEarlyBound(ref br) => {
-                let mut sp = tcx.def_span(scope);
-                if let Some(param) =
+                let scope = region.free_region_binding_scope(tcx).expect_local();
+                let span = if let Some(param) =
                     tcx.hir().get_generics(scope).and_then(|generics| generics.get_named(br.name))
                 {
-                    sp = param.span;
-                }
-                if br.has_name() {
-                    me.kind = "as_defined";
-                    me.arg = br.name.to_string();
+                    param.span
                 } else {
-                    me.kind = "as_defined_anon";
+                    tcx.def_span(scope)
                 };
-                me.span = Some(sp)
+                if br.has_name() {
+                    (Some(span), "as_defined", br.name.to_string())
+                } else {
+                    (Some(span), "as_defined_anon", String::new())
+                }
             }
             ty::ReFree(ref fr) => {
                 if !fr.bound_region.is_named()
                     && let Some((ty, _)) = find_anon_type(tcx, region, &fr.bound_region)
                 {
-                    me.kind = "defined_here";
-                    me.span = Some(ty.span);
+                    (Some(ty.span), "defined_here", String::new())
                 } else {
+                    let scope = region.free_region_binding_scope(tcx).expect_local();
                     match fr.bound_region {
                         ty::BoundRegionKind::BrNamed(_, name) => {
-                            let mut sp = tcx.def_span(scope);
-                            if let Some(param) =
-                                tcx.hir().get_generics(scope).and_then(|generics| generics.get_named(name))
+                            let span = if let Some(param) = tcx
+                                .hir()
+                                .get_generics(scope)
+                                .and_then(|generics| generics.get_named(name))
                             {
-                                sp = param.span;
-                            }
-                            if name == kw::UnderscoreLifetime {
-                                me.kind = "as_defined_anon";
+                                param.span
                             } else {
-                                me.kind = "as_defined";
-                                me.arg = name.to_string();
+                                tcx.def_span(scope)
                             };
-                            me.span = Some(sp);
+                            if name == kw::UnderscoreLifetime {
+                                (Some(span), "as_defined_anon", String::new())
+                            } else {
+                                (Some(span), "as_defined", name.to_string())
+                            }
                         }
-                        ty::BrAnon(idx, span) => {
-                            me.kind = "anon_num_here";
-                            me.num_arg = idx+1;
-                            me.span = match span {
+                        ty::BrAnon(span) => {
+                            let span = match span {
                                 Some(_) => span,
                                 None => Some(tcx.def_span(scope)),
-                            }
-                        },
+                            };
+                            (span, "defined_here", String::new())
+                        }
                         _ => {
-                            me.kind = "defined_here_reg";
-                            me.arg = region.to_string();
-                            me.span = Some(tcx.def_span(scope));
-                        },
+                            (Some(tcx.def_span(scope)), "defined_here_reg", region.to_string())
+                        }
                     }
                 }
             }
-            _ => bug!(),
-        }
-        Some(me)
-    }
 
-    fn add_to(self, diag: &mut rustc_errors::Diagnostic) {
-        diag.set_arg("desc_kind", self.kind);
-        diag.set_arg("desc_arg", self.arg);
-        diag.set_arg("desc_num_arg", self.num_arg);
+            ty::ReStatic => (alt_span, "restatic", String::new()),
+
+            ty::RePlaceholder(_) | ty::ReError(_) => return None,
+
+            // FIXME(#13998) RePlaceholder should probably print like
+            // ReFree rather than dumping Debug output on the user.
+            //
+            // We shouldn't really be having unification failures with ReVar
+            // and ReLateBound though.
+            ty::ReVar(_) | ty::ReLateBound(..) | ty::ReErased => {
+                (alt_span, "revar", format!("{:?}", region))
+            }
+        };
+        Some(DescriptionCtx { span, kind, arg })
     }
 }
 
@@ -199,10 +168,11 @@ impl AddToDiagnostic for RegionExplanation<'_> {
     {
         diag.set_arg("pref_kind", self.prefix);
         diag.set_arg("suff_kind", self.suffix);
-        let desc_span = self.desc.span;
-        self.desc.add_to(diag);
+        diag.set_arg("desc_kind", self.desc.kind);
+        diag.set_arg("desc_arg", self.desc.arg);
+
         let msg = f(diag, fluent::infer_region_explanation.into());
-        if let Some(span) = desc_span {
+        if let Some(span) = self.desc.span {
             diag.span_note(span, msg);
         } else {
             diag.note(msg);
diff --git a/compiler/rustc_infer/src/infer/at.rs b/compiler/rustc_infer/src/infer/at.rs
index 7d9bae735..d240d8e49 100644
--- a/compiler/rustc_infer/src/infer/at.rs
+++ b/compiler/rustc_infer/src/infer/at.rs
@@ -30,16 +30,22 @@ use super::*;
 use rustc_middle::ty::relate::{Relate, TypeRelation};
 use rustc_middle::ty::{Const, ImplSubject};
 
+use std::cell::Cell;
+
+/// Whether we should define opaque types or just treat them opaquely.
+///
+/// Currently only used to prevent predicate matching from matching anything
+/// against opaque types.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum DefineOpaqueTypes {
+    Yes,
+    No,
+}
+
 pub struct At<'a, 'tcx> {
     pub infcx: &'a InferCtxt<'tcx>,
     pub cause: &'a ObligationCause<'tcx>,
     pub param_env: ty::ParamEnv<'tcx>,
-    /// Whether we should define opaque types
-    /// or just treat them opaquely.
-    /// Currently only used to prevent predicate
-    /// matching from matching anything against opaque
-    /// types.
-    pub define_opaque_types: bool,
 }
 
 pub struct Trace<'a, 'tcx> {
@@ -55,7 +61,7 @@ impl<'tcx> InferCtxt<'tcx> {
         cause: &'a ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> At<'a, 'tcx> {
-        At { infcx: self, cause, param_env, define_opaque_types: false }
+        At { infcx: self, cause, param_env }
     }
 
     /// Forks the inference context, creating a new inference context with the same inference
@@ -78,13 +84,13 @@ impl<'tcx> InferCtxt<'tcx> {
             in_snapshot: self.in_snapshot.clone(),
             universe: self.universe.clone(),
             intercrate: self.intercrate,
+            inside_canonicalization_ctxt: Cell::new(self.inside_canonicalization_ctxt()),
         }
     }
 }
 
 pub trait ToTrace<'tcx>: Relate<'tcx> + Copy {
     fn to_trace(
-        tcx: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
@@ -93,33 +99,21 @@ pub trait ToTrace<'tcx>: Relate<'tcx> + Copy {
 }
 
 impl<'a, 'tcx> At<'a, 'tcx> {
-    pub fn define_opaque_types(self, define_opaque_types: bool) -> Self {
-        Self { define_opaque_types, ..self }
-    }
-
-    /// Hacky routine for equating two impl headers in coherence.
-    pub fn eq_impl_headers(
-        self,
-        expected: &ty::ImplHeader<'tcx>,
-        actual: &ty::ImplHeader<'tcx>,
-    ) -> InferResult<'tcx, ()> {
-        debug!("eq_impl_header({:?} = {:?})", expected, actual);
-        match (expected.trait_ref, actual.trait_ref) {
-            (Some(a_ref), Some(b_ref)) => self.eq(a_ref, b_ref),
-            (None, None) => self.eq(expected.self_ty, actual.self_ty),
-            _ => bug!("mk_eq_impl_headers given mismatched impl kinds"),
-        }
-    }
-
     /// Makes `a <: b`, where `a` may or may not be expected.
     ///
     /// See [`At::trace_exp`] and [`Trace::sub`] for a version of
     /// this method that only requires `T: Relate<'tcx>`
-    pub fn sub_exp<T>(self, a_is_expected: bool, a: T, b: T) -> InferResult<'tcx, ()>
+    pub fn sub_exp<T>(
+        self,
+        define_opaque_types: DefineOpaqueTypes,
+        a_is_expected: bool,
+        a: T,
+        b: T,
+    ) -> InferResult<'tcx, ()>
     where
         T: ToTrace<'tcx>,
     {
-        self.trace_exp(a_is_expected, a, b).sub(a, b)
+        self.trace_exp(a_is_expected, a, b).sub(define_opaque_types, a, b)
     }
 
     /// Makes `actual <: expected`. For example, if type-checking a
@@ -129,54 +123,81 @@ impl<'a, 'tcx> At<'a, 'tcx> {
     ///
     /// See [`At::trace`] and [`Trace::sub`] for a version of
     /// this method that only requires `T: Relate<'tcx>`
-    pub fn sup<T>(self, expected: T, actual: T) -> InferResult<'tcx, ()>
+    pub fn sup<T>(
+        self,
+        define_opaque_types: DefineOpaqueTypes,
+        expected: T,
+        actual: T,
+    ) -> InferResult<'tcx, ()>
     where
         T: ToTrace<'tcx>,
     {
-        self.sub_exp(false, actual, expected)
+        self.sub_exp(define_opaque_types, false, actual, expected)
     }
 
     /// Makes `expected <: actual`.
     ///
     /// See [`At::trace`] and [`Trace::sub`] for a version of
     /// this method that only requires `T: Relate<'tcx>`
-    pub fn sub<T>(self, expected: T, actual: T) -> InferResult<'tcx, ()>
+    pub fn sub<T>(
+        self,
+        define_opaque_types: DefineOpaqueTypes,
+        expected: T,
+        actual: T,
+    ) -> InferResult<'tcx, ()>
     where
         T: ToTrace<'tcx>,
     {
-        self.sub_exp(true, expected, actual)
+        self.sub_exp(define_opaque_types, true, expected, actual)
     }
 
     /// Makes `expected <: actual`.
     ///
     /// See [`At::trace_exp`] and [`Trace::eq`] for a version of
     /// this method that only requires `T: Relate<'tcx>`
-    pub fn eq_exp<T>(self, a_is_expected: bool, a: T, b: T) -> InferResult<'tcx, ()>
+    pub fn eq_exp<T>(
+        self,
+        define_opaque_types: DefineOpaqueTypes,
+        a_is_expected: bool,
+        a: T,
+        b: T,
+    ) -> InferResult<'tcx, ()>
     where
         T: ToTrace<'tcx>,
     {
-        self.trace_exp(a_is_expected, a, b).eq(a, b)
+        self.trace_exp(a_is_expected, a, b).eq(define_opaque_types, a, b)
     }
 
     /// Makes `expected <: actual`.
     ///
     /// See [`At::trace`] and [`Trace::eq`] for a version of
     /// this method that only requires `T: Relate<'tcx>`
-    pub fn eq<T>(self, expected: T, actual: T) -> InferResult<'tcx, ()>
+    pub fn eq<T>(
+        self,
+        define_opaque_types: DefineOpaqueTypes,
+        expected: T,
+        actual: T,
+    ) -> InferResult<'tcx, ()>
     where
         T: ToTrace<'tcx>,
     {
-        self.trace(expected, actual).eq(expected, actual)
+        self.trace(expected, actual).eq(define_opaque_types, expected, actual)
     }
 
-    pub fn relate<T>(self, expected: T, variance: ty::Variance, actual: T) -> InferResult<'tcx, ()>
+    pub fn relate<T>(
+        self,
+        define_opaque_types: DefineOpaqueTypes,
+        expected: T,
+        variance: ty::Variance,
+        actual: T,
+    ) -> InferResult<'tcx, ()>
     where
         T: ToTrace<'tcx>,
     {
         match variance {
-            ty::Variance::Covariant => self.sub(expected, actual),
-            ty::Variance::Invariant => self.eq(expected, actual),
-            ty::Variance::Contravariant => self.sup(expected, actual),
+            ty::Variance::Covariant => self.sub(define_opaque_types, expected, actual),
+            ty::Variance::Invariant => self.eq(define_opaque_types, expected, actual),
+            ty::Variance::Contravariant => self.sup(define_opaque_types, expected, actual),
 
             // We could make this make sense but it's not readily
             // exposed and I don't feel like dealing with it. Note
@@ -195,11 +216,16 @@ impl<'a, 'tcx> At<'a, 'tcx> {
     ///
     /// See [`At::trace`] and [`Trace::lub`] for a version of
     /// this method that only requires `T: Relate<'tcx>`
-    pub fn lub<T>(self, expected: T, actual: T) -> InferResult<'tcx, T>
+    pub fn lub<T>(
+        self,
+        define_opaque_types: DefineOpaqueTypes,
+        expected: T,
+        actual: T,
+    ) -> InferResult<'tcx, T>
     where
         T: ToTrace<'tcx>,
     {
-        self.trace(expected, actual).lub(expected, actual)
+        self.trace(expected, actual).lub(define_opaque_types, expected, actual)
     }
 
     /// Computes the greatest-lower-bound, or mutual subtype, of two
@@ -208,11 +234,16 @@ impl<'a, 'tcx> At<'a, 'tcx> {
     ///
     /// See [`At::trace`] and [`Trace::glb`] for a version of
     /// this method that only requires `T: Relate<'tcx>`
-    pub fn glb<T>(self, expected: T, actual: T) -> InferResult<'tcx, T>
+    pub fn glb<T>(
+        self,
+        define_opaque_types: DefineOpaqueTypes,
+        expected: T,
+        actual: T,
+    ) -> InferResult<'tcx, T>
     where
         T: ToTrace<'tcx>,
     {
-        self.trace(expected, actual).glb(expected, actual)
+        self.trace(expected, actual).glb(define_opaque_types, expected, actual)
     }
 
     /// Sets the "trace" values that will be used for
@@ -233,7 +264,7 @@ impl<'a, 'tcx> At<'a, 'tcx> {
     where
         T: ToTrace<'tcx>,
     {
-        let trace = ToTrace::to_trace(self.infcx.tcx, self.cause, a_is_expected, a, b);
+        let trace = ToTrace::to_trace(self.cause, a_is_expected, a, b);
         Trace { at: self, trace, a_is_expected }
     }
 }
@@ -242,13 +273,13 @@ impl<'a, 'tcx> Trace<'a, 'tcx> {
     /// Makes `a <: b` where `a` may or may not be expected (if
     /// `a_is_expected` is true, then `a` is expected).
     #[instrument(skip(self), level = "debug")]
-    pub fn sub<T>(self, a: T, b: T) -> InferResult<'tcx, ()>
+    pub fn sub<T>(self, define_opaque_types: DefineOpaqueTypes, a: T, b: T) -> InferResult<'tcx, ()>
     where
         T: Relate<'tcx>,
     {
         let Trace { at, trace, a_is_expected } = self;
         at.infcx.commit_if_ok(|_| {
-            let mut fields = at.infcx.combine_fields(trace, at.param_env, at.define_opaque_types);
+            let mut fields = at.infcx.combine_fields(trace, at.param_env, define_opaque_types);
             fields
                 .sub(a_is_expected)
                 .relate(a, b)
@@ -259,13 +290,13 @@ impl<'a, 'tcx> Trace<'a, 'tcx> {
     /// Makes `a == b`; the expectation is set by the call to
     /// `trace()`.
     #[instrument(skip(self), level = "debug")]
-    pub fn eq<T>(self, a: T, b: T) -> InferResult<'tcx, ()>
+    pub fn eq<T>(self, define_opaque_types: DefineOpaqueTypes, a: T, b: T) -> InferResult<'tcx, ()>
     where
         T: Relate<'tcx>,
     {
         let Trace { at, trace, a_is_expected } = self;
         at.infcx.commit_if_ok(|_| {
-            let mut fields = at.infcx.combine_fields(trace, at.param_env, at.define_opaque_types);
+            let mut fields = at.infcx.combine_fields(trace, at.param_env, define_opaque_types);
             fields
                 .equate(a_is_expected)
                 .relate(a, b)
@@ -274,13 +305,13 @@ impl<'a, 'tcx> Trace<'a, 'tcx> {
     }
 
     #[instrument(skip(self), level = "debug")]
-    pub fn lub<T>(self, a: T, b: T) -> InferResult<'tcx, T>
+    pub fn lub<T>(self, define_opaque_types: DefineOpaqueTypes, a: T, b: T) -> InferResult<'tcx, T>
     where
         T: Relate<'tcx>,
     {
         let Trace { at, trace, a_is_expected } = self;
         at.infcx.commit_if_ok(|_| {
-            let mut fields = at.infcx.combine_fields(trace, at.param_env, at.define_opaque_types);
+            let mut fields = at.infcx.combine_fields(trace, at.param_env, define_opaque_types);
             fields
                 .lub(a_is_expected)
                 .relate(a, b)
@@ -289,13 +320,13 @@ impl<'a, 'tcx> Trace<'a, 'tcx> {
     }
 
     #[instrument(skip(self), level = "debug")]
-    pub fn glb<T>(self, a: T, b: T) -> InferResult<'tcx, T>
+    pub fn glb<T>(self, define_opaque_types: DefineOpaqueTypes, a: T, b: T) -> InferResult<'tcx, T>
     where
         T: Relate<'tcx>,
     {
         let Trace { at, trace, a_is_expected } = self;
         at.infcx.commit_if_ok(|_| {
-            let mut fields = at.infcx.combine_fields(trace, at.param_env, at.define_opaque_types);
+            let mut fields = at.infcx.combine_fields(trace, at.param_env, define_opaque_types);
             fields
                 .glb(a_is_expected)
                 .relate(a, b)
@@ -306,7 +337,6 @@ impl<'a, 'tcx> Trace<'a, 'tcx> {
 
 impl<'tcx> ToTrace<'tcx> for ImplSubject<'tcx> {
     fn to_trace(
-        tcx: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
@@ -314,10 +344,10 @@ impl<'tcx> ToTrace<'tcx> for ImplSubject<'tcx> {
     ) -> TypeTrace<'tcx> {
         match (a, b) {
             (ImplSubject::Trait(trait_ref_a), ImplSubject::Trait(trait_ref_b)) => {
-                ToTrace::to_trace(tcx, cause, a_is_expected, trait_ref_a, trait_ref_b)
+                ToTrace::to_trace(cause, a_is_expected, trait_ref_a, trait_ref_b)
             }
             (ImplSubject::Inherent(ty_a), ImplSubject::Inherent(ty_b)) => {
-                ToTrace::to_trace(tcx, cause, a_is_expected, ty_a, ty_b)
+                ToTrace::to_trace(cause, a_is_expected, ty_a, ty_b)
             }
             (ImplSubject::Trait(_), ImplSubject::Inherent(_))
             | (ImplSubject::Inherent(_), ImplSubject::Trait(_)) => {
@@ -329,7 +359,6 @@ impl<'tcx> ToTrace<'tcx> for ImplSubject<'tcx> {
 
 impl<'tcx> ToTrace<'tcx> for Ty<'tcx> {
     fn to_trace(
-        _: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
@@ -344,7 +373,6 @@ impl<'tcx> ToTrace<'tcx> for Ty<'tcx> {
 
 impl<'tcx> ToTrace<'tcx> for ty::Region<'tcx> {
     fn to_trace(
-        _: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
@@ -356,7 +384,6 @@ impl<'tcx> ToTrace<'tcx> for ty::Region<'tcx> {
 
 impl<'tcx> ToTrace<'tcx> for Const<'tcx> {
     fn to_trace(
-        _: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
@@ -371,7 +398,6 @@ impl<'tcx> ToTrace<'tcx> for Const<'tcx> {
 
 impl<'tcx> ToTrace<'tcx> for ty::GenericArg<'tcx> {
     fn to_trace(
-        _: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
@@ -399,7 +425,6 @@ impl<'tcx> ToTrace<'tcx> for ty::GenericArg<'tcx> {
 
 impl<'tcx> ToTrace<'tcx> for ty::Term<'tcx> {
     fn to_trace(
-        _: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
@@ -411,7 +436,6 @@ impl<'tcx> ToTrace<'tcx> for ty::Term<'tcx> {
 
 impl<'tcx> ToTrace<'tcx> for ty::TraitRef<'tcx> {
     fn to_trace(
-        _: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
@@ -426,7 +450,6 @@ impl<'tcx> ToTrace<'tcx> for ty::TraitRef<'tcx> {
 
 impl<'tcx> ToTrace<'tcx> for ty::PolyTraitRef<'tcx> {
     fn to_trace(
-        _: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
@@ -441,24 +464,17 @@ impl<'tcx> ToTrace<'tcx> for ty::PolyTraitRef<'tcx> {
 
 impl<'tcx> ToTrace<'tcx> for ty::AliasTy<'tcx> {
     fn to_trace(
-        tcx: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
         b: Self,
     ) -> TypeTrace<'tcx> {
-        let a_ty = tcx.mk_projection(a.def_id, a.substs);
-        let b_ty = tcx.mk_projection(b.def_id, b.substs);
-        TypeTrace {
-            cause: cause.clone(),
-            values: Terms(ExpectedFound::new(a_is_expected, a_ty.into(), b_ty.into())),
-        }
+        TypeTrace { cause: cause.clone(), values: Aliases(ExpectedFound::new(a_is_expected, a, b)) }
     }
 }
 
 impl<'tcx> ToTrace<'tcx> for ty::FnSig<'tcx> {
     fn to_trace(
-        _: TyCtxt<'tcx>,
         cause: &ObligationCause<'tcx>,
         a_is_expected: bool,
         a: Self,
diff --git a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
index 678c4a0be..e808911a3 100644
--- a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
+++ b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
@@ -230,9 +230,9 @@ impl CanonicalizeMode for CanonicalizeUserTypeAnnotation {
         r: ty::Region<'tcx>,
     ) -> ty::Region<'tcx> {
         match *r {
-            ty::ReEarlyBound(_) | ty::ReFree(_) | ty::ReErased | ty::ReStatic => r,
+            ty::ReEarlyBound(_) | ty::ReFree(_) | ty::ReErased | ty::ReStatic | ty::ReError(_) => r,
             ty::ReVar(_) => canonicalizer.canonical_var_for_region_in_root_universe(r),
-            _ => {
+            ty::RePlaceholder(..) | ty::ReLateBound(..) => {
                 // We only expect region names that the user can type.
                 bug!("unexpected region in query response: `{:?}`", r)
             }
@@ -352,19 +352,17 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
             }
 
             ty::ReVar(vid) => {
-                let resolved_vid = self
+                let resolved = self
                     .infcx
                     .inner
                     .borrow_mut()
                     .unwrap_region_constraints()
-                    .opportunistic_resolve_var(vid);
+                    .opportunistic_resolve_var(self.tcx, vid);
                 debug!(
-                    "canonical: region var found with vid {:?}, \
-                     opportunistically resolved to {:?}",
-                    vid, resolved_vid
+                    "canonical: region var found with vid {vid:?}, \
+                     opportunistically resolved to {resolved:?}",
                 );
-                let r = self.tcx.mk_re_var(resolved_vid);
-                self.canonicalize_mode.canonicalize_free_region(self, r)
+                self.canonicalize_mode.canonicalize_free_region(self, resolved)
             }
 
             ty::ReStatic
@@ -376,9 +374,18 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
         }
     }
 
-    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
+    fn fold_ty(&mut self, mut t: Ty<'tcx>) -> Ty<'tcx> {
         match *t.kind() {
-            ty::Infer(ty::TyVar(vid)) => {
+            ty::Infer(ty::TyVar(mut vid)) => {
+                // We need to canonicalize the *root* of our ty var.
+                // This is so that our canonical response correctly reflects
+                // any equated inference vars correctly!
+                let root_vid = self.infcx.root_var(vid);
+                if root_vid != vid {
+                    t = self.infcx.tcx.mk_ty_var(root_vid);
+                    vid = root_vid;
+                }
+
                 debug!("canonical: type var found with vid {:?}", vid);
                 match self.infcx.probe_ty_var(vid) {
                     // `t` could be a float / int variable; canonicalize that instead.
@@ -404,15 +411,28 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 }
             }
 
-            ty::Infer(ty::IntVar(_)) => self.canonicalize_ty_var(
-                CanonicalVarInfo { kind: CanonicalVarKind::Ty(CanonicalTyVarKind::Int) },
-                t,
-            ),
-
-            ty::Infer(ty::FloatVar(_)) => self.canonicalize_ty_var(
-                CanonicalVarInfo { kind: CanonicalVarKind::Ty(CanonicalTyVarKind::Float) },
-                t,
-            ),
+            ty::Infer(ty::IntVar(vid)) => {
+                let nt = self.infcx.opportunistic_resolve_int_var(vid);
+                if nt != t {
+                    return self.fold_ty(nt);
+                } else {
+                    self.canonicalize_ty_var(
+                        CanonicalVarInfo { kind: CanonicalVarKind::Ty(CanonicalTyVarKind::Int) },
+                        t,
+                    )
+                }
+            }
+            ty::Infer(ty::FloatVar(vid)) => {
+                let nt = self.infcx.opportunistic_resolve_float_var(vid);
+                if nt != t {
+                    return self.fold_ty(nt);
+                } else {
+                    self.canonicalize_ty_var(
+                        CanonicalVarInfo { kind: CanonicalVarKind::Ty(CanonicalTyVarKind::Float) },
+                        t,
+                    )
+                }
+            }
 
             ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
                 bug!("encountered a fresh type during canonicalization")
@@ -469,9 +489,18 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
         }
     }
 
-    fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
+    fn fold_const(&mut self, mut ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
         match ct.kind() {
-            ty::ConstKind::Infer(InferConst::Var(vid)) => {
+            ty::ConstKind::Infer(InferConst::Var(mut vid)) => {
+                // We need to canonicalize the *root* of our const var.
+                // This is so that our canonical response correctly reflects
+                // any equated inference vars correctly!
+                let root_vid = self.infcx.root_const_var(vid);
+                if root_vid != vid {
+                    ct = self.infcx.tcx.mk_const(ty::InferConst::Var(root_vid), ct.ty());
+                    vid = root_vid;
+                }
+
                 debug!("canonical: const var found with vid {:?}", vid);
                 match self.infcx.probe_const_var(vid) {
                     Ok(c) => {
@@ -532,6 +561,8 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     where
         V: TypeFoldable<TyCtxt<'tcx>>,
     {
+        let _inside_canonical_ctxt_guard = infcx.set_canonicalization_ctxt();
+
         let needs_canonical_flags = if canonicalize_region_mode.any() {
             TypeFlags::NEEDS_INFER |
             TypeFlags::HAS_FREE_REGIONS | // `HAS_RE_PLACEHOLDER` implies `HAS_FREE_REGIONS`
@@ -741,7 +772,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         r: ty::Region<'tcx>,
     ) -> ty::Region<'tcx> {
         let var = self.canonical_var(info, r.into());
-        let br = ty::BoundRegion { var, kind: ty::BrAnon(var.as_u32(), None) };
+        let br = ty::BoundRegion { var, kind: ty::BrAnon(None) };
         self.interner().mk_re_late_bound(self.binder_index, br)
     }
 
diff --git a/compiler/rustc_infer/src/infer/canonical/mod.rs b/compiler/rustc_infer/src/infer/canonical/mod.rs
index ce230afda..fbb2257bf 100644
--- a/compiler/rustc_infer/src/infer/canonical/mod.rs
+++ b/compiler/rustc_infer/src/infer/canonical/mod.rs
@@ -125,9 +125,9 @@ impl<'tcx> InferCtxt<'tcx> {
                 ty.into()
             }
 
-            CanonicalVarKind::PlaceholderTy(ty::PlaceholderType { universe, name }) => {
+            CanonicalVarKind::PlaceholderTy(ty::PlaceholderType { universe, bound }) => {
                 let universe_mapped = universe_map(universe);
-                let placeholder_mapped = ty::PlaceholderType { universe: universe_mapped, name };
+                let placeholder_mapped = ty::PlaceholderType { universe: universe_mapped, bound };
                 self.tcx.mk_placeholder(placeholder_mapped).into()
             }
 
@@ -138,9 +138,9 @@ impl<'tcx> InferCtxt<'tcx> {
                 )
                 .into(),
 
-            CanonicalVarKind::PlaceholderRegion(ty::PlaceholderRegion { universe, name }) => {
+            CanonicalVarKind::PlaceholderRegion(ty::PlaceholderRegion { universe, bound }) => {
                 let universe_mapped = universe_map(universe);
-                let placeholder_mapped = ty::PlaceholderRegion { universe: universe_mapped, name };
+                let placeholder_mapped = ty::PlaceholderRegion { universe: universe_mapped, bound };
                 self.tcx.mk_re_placeholder(placeholder_mapped).into()
             }
 
@@ -152,9 +152,9 @@ impl<'tcx> InferCtxt<'tcx> {
                 )
                 .into(),
 
-            CanonicalVarKind::PlaceholderConst(ty::PlaceholderConst { universe, name }, ty) => {
+            CanonicalVarKind::PlaceholderConst(ty::PlaceholderConst { universe, bound }, ty) => {
                 let universe_mapped = universe_map(universe);
-                let placeholder_mapped = ty::PlaceholderConst { universe: universe_mapped, name };
+                let placeholder_mapped = ty::PlaceholderConst { universe: universe_mapped, bound };
                 self.tcx.mk_const(placeholder_mapped, ty).into()
             }
         }
diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs
index 436d29c24..e98f68ae5 100644
--- a/compiler/rustc_infer/src/infer/canonical/query_response.rs
+++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs
@@ -14,7 +14,7 @@ use crate::infer::canonical::{
 };
 use crate::infer::nll_relate::{TypeRelating, TypeRelatingDelegate};
 use crate::infer::region_constraints::{Constraint, RegionConstraintData};
-use crate::infer::{InferCtxt, InferOk, InferResult, NllRegionVariableOrigin};
+use crate::infer::{DefineOpaqueTypes, InferCtxt, InferOk, InferResult, NllRegionVariableOrigin};
 use crate::traits::query::{Fallible, NoSolution};
 use crate::traits::{Obligation, ObligationCause, PredicateObligation};
 use crate::traits::{PredicateObligations, TraitEngine, TraitEngineExt};
@@ -159,9 +159,7 @@ impl<'tcx> InferCtxt<'tcx> {
             .opaque_type_storage
             .opaque_types
             .iter()
-            .map(|&(k, ref v)| {
-                (self.tcx.mk_opaque(k.def_id.to_def_id(), k.substs), v.hidden_type.ty)
-            })
+            .map(|(k, v)| (self.tcx.mk_opaque(k.def_id.to_def_id(), k.substs), v.hidden_type.ty))
             .collect()
     }
 
@@ -510,7 +508,7 @@ impl<'tcx> InferCtxt<'tcx> {
             let b = substitute_value(self.tcx, &result_subst, b);
             debug!(?a, ?b, "constrain opaque type");
             obligations
-                .extend(self.at(cause, param_env).define_opaque_types(true).eq(a, b)?.obligations);
+                .extend(self.at(cause, param_env).eq(DefineOpaqueTypes::Yes, a, b)?.obligations);
         }
 
         Ok(InferOk { value: result_subst, obligations })
@@ -603,8 +601,11 @@ impl<'tcx> InferCtxt<'tcx> {
 
                 match (value1.unpack(), value2.unpack()) {
                     (GenericArgKind::Type(v1), GenericArgKind::Type(v2)) => {
-                        obligations
-                            .extend(self.at(cause, param_env).eq(v1, v2)?.into_obligations());
+                        obligations.extend(
+                            self.at(cause, param_env)
+                                .eq(DefineOpaqueTypes::Yes, v1, v2)?
+                                .into_obligations(),
+                        );
                     }
                     (GenericArgKind::Lifetime(re1), GenericArgKind::Lifetime(re2))
                         if re1.is_erased() && re2.is_erased() =>
@@ -612,11 +613,14 @@ impl<'tcx> InferCtxt<'tcx> {
                         // no action needed
                     }
                     (GenericArgKind::Lifetime(v1), GenericArgKind::Lifetime(v2)) => {
-                        obligations
-                            .extend(self.at(cause, param_env).eq(v1, v2)?.into_obligations());
+                        obligations.extend(
+                            self.at(cause, param_env)
+                                .eq(DefineOpaqueTypes::Yes, v1, v2)?
+                                .into_obligations(),
+                        );
                     }
                     (GenericArgKind::Const(v1), GenericArgKind::Const(v2)) => {
-                        let ok = self.at(cause, param_env).eq(v1, v2)?;
+                        let ok = self.at(cause, param_env).eq(DefineOpaqueTypes::Yes, v1, v2)?;
                         obligations.extend(ok.into_obligations());
                     }
                     _ => {
@@ -636,11 +640,9 @@ pub fn make_query_region_constraints<'tcx>(
     outlives_obligations: impl Iterator<Item = (Ty<'tcx>, ty::Region<'tcx>, ConstraintCategory<'tcx>)>,
     region_constraints: &RegionConstraintData<'tcx>,
 ) -> QueryRegionConstraints<'tcx> {
-    let RegionConstraintData { constraints, verifys, givens, member_constraints } =
-        region_constraints;
+    let RegionConstraintData { constraints, verifys, member_constraints } = region_constraints;
 
     assert!(verifys.is_empty());
-    assert!(givens.is_empty());
 
     debug!(?constraints);
 
diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs
index 33292e871..fe45b5ebe 100644
--- a/compiler/rustc_infer/src/infer/combine.rs
+++ b/compiler/rustc_infer/src/infer/combine.rs
@@ -27,14 +27,13 @@ use super::glb::Glb;
 use super::lub::Lub;
 use super::sub::Sub;
 use super::type_variable::TypeVariableValue;
-use super::{InferCtxt, MiscVariable, TypeTrace};
+use super::{DefineOpaqueTypes, InferCtxt, MiscVariable, TypeTrace};
 use crate::traits::{Obligation, PredicateObligations};
 use rustc_data_structures::sso::SsoHashMap;
 use rustc_hir::def_id::DefId;
 use rustc_middle::infer::canonical::OriginalQueryValues;
 use rustc_middle::infer::unify_key::{ConstVarValue, ConstVariableValue};
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
-use rustc_middle::traits::query::NoSolution;
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::relate::{self, Relate, RelateResult, TypeRelation};
@@ -53,12 +52,7 @@ pub struct CombineFields<'infcx, 'tcx> {
     pub cause: Option<ty::relate::Cause>,
     pub param_env: ty::ParamEnv<'tcx>,
     pub obligations: PredicateObligations<'tcx>,
-    /// Whether we should define opaque types
-    /// or just treat them opaquely.
-    /// Currently only used to prevent predicate
-    /// matching from matching anything against opaque
-    /// types.
-    pub define_opaque_types: bool,
+    pub define_opaque_types: DefineOpaqueTypes,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -119,17 +113,39 @@ impl<'tcx> InferCtxt<'tcx> {
                 self.unify_float_variable(!a_is_expected, v_id, v)
             }
 
+            // We don't expect `TyVar` or `Fresh*` vars at this point with lazy norm.
+            (
+                ty::Alias(AliasKind::Projection, _),
+                ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)),
+            )
+            | (
+                ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)),
+                ty::Alias(AliasKind::Projection, _),
+            ) if self.tcx.trait_solver_next() => {
+                bug!()
+            }
+
+            (_, ty::Alias(AliasKind::Projection, _)) | (ty::Alias(AliasKind::Projection, _), _)
+                if self.tcx.trait_solver_next() =>
+            {
+                relation.register_type_relate_obligation(a, b);
+                Ok(a)
+            }
+
             // All other cases of inference are errors
             (&ty::Infer(_), _) | (_, &ty::Infer(_)) => {
                 Err(TypeError::Sorts(ty::relate::expected_found(relation, a, b)))
             }
 
-            (ty::Alias(AliasKind::Projection, _), _) if self.tcx.trait_solver_next() => {
-                relation.register_type_equate_obligation(a, b);
-                Ok(b)
-            }
-            (_, ty::Alias(AliasKind::Projection, _)) if self.tcx.trait_solver_next() => {
-                relation.register_type_equate_obligation(b, a);
+            // During coherence, opaque types should be treated as *possibly*
+            // equal to each other, even if their generic params differ, as
+            // they could resolve to the same hidden type, even for different
+            // generic params.
+            (
+                &ty::Alias(ty::Opaque, ty::AliasTy { def_id: a_def_id, .. }),
+                &ty::Alias(ty::Opaque, ty::AliasTy { def_id: b_def_id, .. }),
+            ) if self.intercrate && a_def_id == b_def_id => {
+                relation.register_predicates([ty::Binder::dummy(ty::PredicateKind::Ambiguous)]);
                 Ok(a)
             }
 
@@ -161,9 +177,9 @@ impl<'tcx> InferCtxt<'tcx> {
         //
         // This probe is probably not strictly necessary but it seems better to be safe and not accidentally find
         // ourselves with a check to find bugs being required for code to compile because it made inference progress.
-        self.probe(|_| {
+        let compatible_types = self.probe(|_| {
             if a.ty() == b.ty() {
-                return;
+                return Ok(());
             }
 
             // We don't have access to trait solving machinery in `rustc_infer` so the logic for determining if the
@@ -173,15 +189,33 @@ impl<'tcx> InferCtxt<'tcx> {
                 (relation.param_env(), a.ty(), b.ty()),
                 &mut OriginalQueryValues::default(),
             );
-
-            if let Err(NoSolution) = self.tcx.check_tys_might_be_eq(canonical) {
+            self.tcx.check_tys_might_be_eq(canonical).map_err(|_| {
                 self.tcx.sess.delay_span_bug(
                     DUMMY_SP,
                     &format!("cannot relate consts of different types (a={:?}, b={:?})", a, b,),
-                );
-            }
+                )
+            })
         });
 
+        // If the consts have differing types, just bail with a const error with
+        // the expected const's type. Specifically, we don't want const infer vars
+        // to do any type shapeshifting before and after resolution.
+        if let Err(guar) = compatible_types {
+            // HACK: equating both sides with `[const error]` eagerly prevents us
+            // from leaving unconstrained inference vars during things like impl
+            // matching in the solver.
+            let a_error = self.tcx.const_error_with_guaranteed(a.ty(), guar);
+            if let ty::ConstKind::Infer(InferConst::Var(vid)) = a.kind() {
+                return self.unify_const_variable(vid, a_error);
+            }
+            let b_error = self.tcx.const_error_with_guaranteed(b.ty(), guar);
+            if let ty::ConstKind::Infer(InferConst::Var(vid)) = b.kind() {
+                return self.unify_const_variable(vid, b_error);
+            }
+
+            return Ok(if relation.a_is_expected() { a_error } else { b_error });
+        }
+
         match (a.kind(), b.kind()) {
             (
                 ty::ConstKind::Infer(InferConst::Var(a_vid)),
@@ -483,10 +517,6 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
             Obligation::new(self.infcx.tcx, self.trace.cause.clone(), self.param_env, to_pred)
         }))
     }
-
-    pub fn mark_ambiguous(&mut self) {
-        self.register_predicates([ty::Binder::dummy(ty::PredicateKind::Ambiguous)]);
-    }
 }
 
 struct Generalizer<'cx, 'tcx> {
@@ -559,10 +589,6 @@ impl<'tcx> TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
         self.infcx.tcx
     }
 
-    fn intercrate(&self) -> bool {
-        self.infcx.intercrate
-    }
-
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
         self.param_env
     }
@@ -575,10 +601,6 @@ impl<'tcx> TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
         true
     }
 
-    fn mark_ambiguous(&mut self) {
-        span_bug!(self.cause.span, "opaque types are handled in `tys`");
-    }
-
     fn binders<T>(
         &mut self,
         a: ty::Binder<'tcx, T>,
@@ -820,23 +842,25 @@ pub trait ObligationEmittingRelation<'tcx>: TypeRelation<'tcx> {
         let (a, b) = if self.a_is_expected() { (a, b) } else { (b, a) };
 
         self.register_predicates([ty::Binder::dummy(if self.tcx().trait_solver_next() {
-            ty::PredicateKind::AliasEq(a.into(), b.into())
+            ty::PredicateKind::AliasRelate(a.into(), b.into(), ty::AliasRelationDirection::Equate)
         } else {
             ty::PredicateKind::ConstEquate(a, b)
         })]);
     }
 
-    /// Register an obligation that both types must be equal to each other.
-    ///
-    /// If they aren't equal then the relation doesn't hold.
-    fn register_type_equate_obligation(&mut self, a: Ty<'tcx>, b: Ty<'tcx>) {
-        let (a, b) = if self.a_is_expected() { (a, b) } else { (b, a) };
-
-        self.register_predicates([ty::Binder::dummy(ty::PredicateKind::AliasEq(
+    /// Register an obligation that both types must be related to each other according to
+    /// the [`ty::AliasRelationDirection`] given by [`ObligationEmittingRelation::alias_relate_direction`]
+    fn register_type_relate_obligation(&mut self, a: Ty<'tcx>, b: Ty<'tcx>) {
+        self.register_predicates([ty::Binder::dummy(ty::PredicateKind::AliasRelate(
             a.into(),
             b.into(),
+            self.alias_relate_direction(),
         ))]);
     }
+
+    /// Relation direction emitted for `AliasRelate` predicates, corresponding to the direction
+    /// of the relation.
+    fn alias_relate_direction(&self) -> ty::AliasRelationDirection;
 }
 
 fn int_unification_error<'tcx>(
diff --git a/compiler/rustc_infer/src/infer/equate.rs b/compiler/rustc_infer/src/infer/equate.rs
index 54a62326e..fe4a2dd38 100644
--- a/compiler/rustc_infer/src/infer/equate.rs
+++ b/compiler/rustc_infer/src/infer/equate.rs
@@ -1,3 +1,4 @@
+use crate::infer::DefineOpaqueTypes;
 use crate::traits::PredicateObligations;
 
 use super::combine::{CombineFields, ObligationEmittingRelation, RelationDir};
@@ -34,10 +35,6 @@ impl<'tcx> TypeRelation<'tcx> for Equate<'_, '_, 'tcx> {
         self.fields.tcx()
     }
 
-    fn intercrate(&self) -> bool {
-        self.fields.infcx.intercrate
-    }
-
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
         self.fields.param_env
     }
@@ -46,10 +43,6 @@ impl<'tcx> TypeRelation<'tcx> for Equate<'_, '_, 'tcx> {
         self.a_is_expected
     }
 
-    fn mark_ambiguous(&mut self) {
-        self.fields.mark_ambiguous();
-    }
-
     fn relate_item_substs(
         &mut self,
         _item_def_id: DefId,
@@ -110,7 +103,8 @@ impl<'tcx> TypeRelation<'tcx> for Equate<'_, '_, 'tcx> {
             }
             (&ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }), _)
             | (_, &ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }))
-                if self.fields.define_opaque_types && def_id.is_local() =>
+                if self.fields.define_opaque_types == DefineOpaqueTypes::Yes
+                    && def_id.is_local() =>
             {
                 self.fields.obligations.extend(
                     infcx
@@ -208,4 +202,8 @@ impl<'tcx> ObligationEmittingRelation<'tcx> for Equate<'_, '_, 'tcx> {
     fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
         self.fields.register_obligations(obligations);
     }
+
+    fn alias_relate_direction(&self) -> ty::AliasRelationDirection {
+        ty::AliasRelationDirection::Equate
+    }
 }
diff --git a/compiler/rustc_infer/src/infer/error_reporting/mod.rs b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
index 8a2b800af..9e5f6d107 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/mod.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
@@ -49,11 +49,10 @@ use super::lexical_region_resolve::RegionResolutionError;
 use super::region_constraints::GenericKind;
 use super::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TypeTrace, ValuePairs};
 
-use crate::errors;
+use crate::errors::{self, ObligationCauseFailureCode, TypeErrorAdditionalDiags};
 use crate::infer;
 use crate::infer::error_reporting::nice_region_error::find_anon_type::find_anon_type;
 use crate::infer::ExpectedFound;
-use crate::traits::error_reporting::report_object_safety_error;
 use crate::traits::{
     IfExpressionCause, MatchExpressionArmCause, ObligationCause, ObligationCauseCode,
     PredicateObligation,
@@ -75,6 +74,7 @@ use rustc_middle::ty::{
     self, error::TypeError, List, Region, Ty, TyCtxt, TypeFoldable, TypeSuperVisitable,
     TypeVisitable, TypeVisitableExt,
 };
+use rustc_span::DUMMY_SP;
 use rustc_span::{sym, symbol::kw, BytePos, DesugaringKind, Pos, Span};
 use rustc_target::spec::abi;
 use std::ops::{ControlFlow, Deref};
@@ -90,9 +90,35 @@ pub use need_type_info::TypeAnnotationNeeded;
 
 pub mod nice_region_error;
 
+/// Makes a valid string literal from a string by escaping special characters (" and \),
+/// unless they are already escaped.
+fn escape_literal(s: &str) -> String {
+    let mut escaped = String::with_capacity(s.len());
+    let mut chrs = s.chars().peekable();
+    while let Some(first) = chrs.next() {
+        match (first, chrs.peek()) {
+            ('\\', Some(&delim @ '"') | Some(&delim @ '\'')) => {
+                escaped.push('\\');
+                escaped.push(delim);
+                chrs.next();
+            }
+            ('"' | '\'', _) => {
+                escaped.push('\\');
+                escaped.push(first)
+            }
+            (c, _) => escaped.push(c),
+        };
+    }
+    escaped
+}
+
 /// A helper for building type related errors. The `typeck_results`
 /// field is only populated during an in-progress typeck.
-/// Get an instance by calling `InferCtxt::err` or `FnCtxt::infer_err`.
+/// Get an instance by calling `InferCtxt::err_ctxt` or `FnCtxt::err_ctxt`.
+///
+/// You must only create this if you intend to actually emit an error.
+/// This provides a lot of utility methods which should not be used
+/// during the happy path.
 pub struct TypeErrCtxt<'a, 'tcx> {
     pub infcx: &'a InferCtxt<'tcx>,
     pub typeck_results: Option<std::cell::Ref<'a, ty::TypeckResults<'tcx>>>,
@@ -104,6 +130,19 @@ pub struct TypeErrCtxt<'a, 'tcx> {
         Box<dyn Fn(Ty<'tcx>) -> Vec<(Ty<'tcx>, Vec<PredicateObligation<'tcx>>)> + 'a>,
 }
 
+impl Drop for TypeErrCtxt<'_, '_> {
+    fn drop(&mut self) {
+        if let Some(_) = self.infcx.tcx.sess.has_errors_or_delayed_span_bugs() {
+            // ok, emitted an error.
+        } else {
+            self.infcx
+                .tcx
+                .sess
+                .delay_span_bug(DUMMY_SP, "used a `TypeErrCtxt` without failing compilation");
+        }
+    }
+}
+
 impl TypeErrCtxt<'_, '_> {
     /// This is just to avoid a potential footgun of accidentally
     /// dropping `typeck_results` by calling `InferCtxt::err_ctxt`
@@ -164,83 +203,73 @@ fn msg_span_from_named_region<'tcx>(
     alt_span: Option<Span>,
 ) -> (String, Option<Span>) {
     match *region {
-        ty::ReEarlyBound(_) | ty::ReFree(_) => {
-            let (msg, span) = msg_span_from_early_bound_and_free_regions(tcx, region);
-            (msg, Some(span))
-        }
-        ty::ReStatic => ("the static lifetime".to_owned(), alt_span),
-        ty::RePlaceholder(ty::PlaceholderRegion {
-            name: ty::BoundRegionKind::BrNamed(def_id, name),
-            ..
-        }) => (format!("the lifetime `{name}` as defined here"), Some(tcx.def_span(def_id))),
-        ty::RePlaceholder(ty::PlaceholderRegion {
-            name: ty::BoundRegionKind::BrAnon(_, Some(span)),
-            ..
-        }) => (format!("the anonymous lifetime defined here"), Some(span)),
-        ty::RePlaceholder(ty::PlaceholderRegion {
-            name: ty::BoundRegionKind::BrAnon(_, None),
-            ..
-        }) => (format!("an anonymous lifetime"), None),
-        _ => bug!("{:?}", region),
-    }
-}
-
-fn msg_span_from_early_bound_and_free_regions<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    region: ty::Region<'tcx>,
-) -> (String, Span) {
-    let scope = region.free_region_binding_scope(tcx).expect_local();
-    match *region {
         ty::ReEarlyBound(ref br) => {
-            let mut sp = tcx.def_span(scope);
-            if let Some(param) =
+            let scope = region.free_region_binding_scope(tcx).expect_local();
+            let span = if let Some(param) =
                 tcx.hir().get_generics(scope).and_then(|generics| generics.get_named(br.name))
             {
-                sp = param.span;
-            }
+                param.span
+            } else {
+                tcx.def_span(scope)
+            };
             let text = if br.has_name() {
                 format!("the lifetime `{}` as defined here", br.name)
             } else {
                 "the anonymous lifetime as defined here".to_string()
             };
-            (text, sp)
+            (text, Some(span))
         }
         ty::ReFree(ref fr) => {
             if !fr.bound_region.is_named()
                 && let Some((ty, _)) = find_anon_type(tcx, region, &fr.bound_region)
             {
-                ("the anonymous lifetime defined here".to_string(), ty.span)
+                ("the anonymous lifetime defined here".to_string(), Some(ty.span))
             } else {
+                let scope = region.free_region_binding_scope(tcx).expect_local();
                 match fr.bound_region {
                     ty::BoundRegionKind::BrNamed(_, name) => {
-                        let mut sp = tcx.def_span(scope);
-                        if let Some(param) =
+                        let span = if let Some(param) =
                             tcx.hir().get_generics(scope).and_then(|generics| generics.get_named(name))
                         {
-                            sp = param.span;
-                        }
+                            param.span
+                        } else {
+                            tcx.def_span(scope)
+                        };
                         let text = if name == kw::UnderscoreLifetime {
                             "the anonymous lifetime as defined here".to_string()
                         } else {
                             format!("the lifetime `{}` as defined here", name)
                         };
-                        (text, sp)
+                        (text, Some(span))
                     }
-                    ty::BrAnon(idx, span) => (
-                        format!("the anonymous lifetime #{} defined here", idx + 1),
-                        match span {
+                    ty::BrAnon(span) => (
+                        "the anonymous lifetime as defined here".to_string(),
+                        Some(match span {
                             Some(span) => span,
                             None => tcx.def_span(scope)
-                        }
+                        })
                     ),
                     _ => (
                         format!("the lifetime `{}` as defined here", region),
-                        tcx.def_span(scope),
+                        Some(tcx.def_span(scope)),
                     ),
                 }
             }
         }
-        _ => bug!(),
+        ty::ReStatic => ("the static lifetime".to_owned(), alt_span),
+        ty::RePlaceholder(ty::PlaceholderRegion {
+            bound: ty::BoundRegion { kind: ty::BoundRegionKind::BrNamed(def_id, name), .. },
+            ..
+        }) => (format!("the lifetime `{name}` as defined here"), Some(tcx.def_span(def_id))),
+        ty::RePlaceholder(ty::PlaceholderRegion {
+            bound: ty::BoundRegion { kind: ty::BoundRegionKind::BrAnon(Some(span)), .. },
+            ..
+        }) => (format!("the anonymous lifetime defined here"), Some(span)),
+        ty::RePlaceholder(ty::PlaceholderRegion {
+            bound: ty::BoundRegion { kind: ty::BoundRegionKind::BrAnon(None), .. },
+            ..
+        }) => (format!("an anonymous lifetime"), None),
+        _ => bug!("{:?}", region),
     }
 }
 
@@ -359,10 +388,12 @@ impl<'tcx> InferCtxt<'tcx> {
     pub fn get_impl_future_output_ty(&self, ty: Ty<'tcx>) -> Option<Ty<'tcx>> {
         let (def_id, substs) = match *ty.kind() {
             ty::Alias(_, ty::AliasTy { def_id, substs, .. })
-                if matches!(
-                    self.tcx.def_kind(def_id),
-                    DefKind::OpaqueTy | DefKind::ImplTraitPlaceholder
-                ) =>
+                if matches!(self.tcx.def_kind(def_id), DefKind::OpaqueTy) =>
+            {
+                (def_id, substs)
+            }
+            ty::Alias(_, ty::AliasTy { def_id, substs, .. })
+                if self.tcx.is_impl_trait_in_trait(def_id) =>
             {
                 (def_id, substs)
             }
@@ -396,7 +427,11 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         &self,
         generic_param_scope: LocalDefId,
         errors: &[RegionResolutionError<'tcx>],
-    ) {
+    ) -> ErrorGuaranteed {
+        if let Some(guaranteed) = self.infcx.tainted_by_errors() {
+            return guaranteed;
+        }
+
         debug!("report_region_errors(): {} errors to start", errors.len());
 
         // try to pre-process the errors, which will group some of them
@@ -476,6 +511,10 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 }
             }
         }
+
+        self.tcx
+            .sess
+            .delay_span_bug(self.tcx.def_span(generic_param_scope), "expected region errors")
     }
 
     // This method goes through all the errors and try to group certain types
@@ -613,9 +652,10 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         }
 
         let report_path_match = |err: &mut Diagnostic, did1: DefId, did2: DefId| {
-            // Only external crates, if either is from a local
-            // module we could have false positives
-            if !(did1.is_local() || did2.is_local()) && did1.krate != did2.krate {
+            // Only report definitions from different crates. If both definitions
+            // are from a local module we could have false positives, e.g.
+            // let _ = [{struct Foo; Foo}, {struct Foo; Foo}];
+            if did1.krate != did2.krate {
                 let abs_path =
                     |def_id| AbsolutePathPrinter { tcx: self.tcx }.print_def_path(def_id, &[]);
 
@@ -627,10 +667,16 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 };
                 if same_path().unwrap_or(false) {
                     let crate_name = self.tcx.crate_name(did1.krate);
-                    err.note(&format!(
-                        "perhaps two different versions of crate `{}` are being used?",
-                        crate_name
-                    ));
+                    let msg = if did1.is_local() || did2.is_local() {
+                        format!(
+                            "the crate `{crate_name}` is compiled multiple times, possibly with different configurations"
+                        )
+                    } else {
+                        format!(
+                            "perhaps two different versions of crate `{crate_name}` are being used?"
+                        )
+                    };
+                    err.note(msg);
                 }
             }
         };
@@ -969,7 +1015,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             let (_, sig, reg) = ty::print::FmtPrinter::new(self.tcx, Namespace::TypeNS)
                 .name_all_regions(sig)
                 .unwrap();
-            let lts: Vec<String> = reg.into_iter().map(|(_, kind)| kind.to_string()).collect();
+            let lts: Vec<String> = reg.into_values().map(|kind| kind.to_string()).collect();
             (if lts.is_empty() { String::new() } else { format!("for<{}> ", lts.join(", ")) }, sig)
         };
 
@@ -1568,6 +1614,9 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     ValuePairs::TraitRefs(_) | ValuePairs::PolyTraitRefs(_) => {
                         (false, Mismatch::Fixed("trait"))
                     }
+                    ValuePairs::Aliases(infer::ExpectedFound { expected, .. }) => {
+                        (false, Mismatch::Fixed(self.tcx.def_descr(expected.def_id)))
+                    }
                     ValuePairs::Regions(_) => (false, Mismatch::Fixed("lifetime")),
                 };
                 let Some(vals) = self.values_str(values) else {
@@ -1754,8 +1803,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                                 )
                             }
                             (true, ty::Alias(ty::Projection, proj))
-                                if self.tcx.def_kind(proj.def_id)
-                                    == DefKind::ImplTraitPlaceholder =>
+                                if self.tcx.is_impl_trait_in_trait(proj.def_id) =>
                             {
                                 let sm = self.tcx.sess.source_map();
                                 let pos = sm.lookup_char_pos(self.tcx.def_span(proj.def_id).lo());
@@ -1797,7 +1845,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                             // will try to hide in some case such as `async fn`, so
                             // to make an error more use friendly we will
                             // avoid to suggest a mismatch type with a
-                            // type that the user usually are not usign
+                            // type that the user usually are not using
                             // directly such as `impl Future<Output = u8>`.
                             if !self.tcx.ty_is_opaque_future(found_ty) {
                                 diag.note_expected_found_extra(
@@ -1888,232 +1936,182 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         debug!(?diag);
     }
 
-    pub fn report_and_explain_type_error(
+    pub fn type_error_additional_suggestions(
         &self,
-        trace: TypeTrace<'tcx>,
+        trace: &TypeTrace<'tcx>,
         terr: TypeError<'tcx>,
-    ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
+    ) -> Vec<TypeErrorAdditionalDiags> {
         use crate::traits::ObligationCauseCode::MatchExpressionArm;
-
-        debug!("report_and_explain_type_error(trace={:?}, terr={:?})", trace, terr);
-
+        let mut suggestions = Vec::new();
         let span = trace.cause.span();
-        let failure_code = trace.cause.as_failure_code(terr);
-        let mut diag = match failure_code {
-            FailureCode::Error0038(did) => {
-                let violations = self.tcx.object_safety_violations(did);
-                report_object_safety_error(self.tcx, span, did, violations)
-            }
-            FailureCode::Error0317(failure_str) => {
-                struct_span_err!(self.tcx.sess, span, E0317, "{}", failure_str)
-            }
-            FailureCode::Error0580(failure_str) => {
-                struct_span_err!(self.tcx.sess, span, E0580, "{}", failure_str)
-            }
-            FailureCode::Error0308(failure_str) => {
-                fn escape_literal(s: &str) -> String {
-                    let mut escaped = String::with_capacity(s.len());
-                    let mut chrs = s.chars().peekable();
-                    while let Some(first) = chrs.next() {
-                        match (first, chrs.peek()) {
-                            ('\\', Some(&delim @ '"') | Some(&delim @ '\'')) => {
-                                escaped.push('\\');
-                                escaped.push(delim);
-                                chrs.next();
-                            }
-                            ('"' | '\'', _) => {
-                                escaped.push('\\');
-                                escaped.push(first)
-                            }
-                            (c, _) => escaped.push(c),
-                        };
+        let values = self.resolve_vars_if_possible(trace.values);
+        if let Some((expected, found)) = values.ty() {
+            match (expected.kind(), found.kind()) {
+                (ty::Tuple(_), ty::Tuple(_)) => {}
+                // If a tuple of length one was expected and the found expression has
+                // parentheses around it, perhaps the user meant to write `(expr,)` to
+                // build a tuple (issue #86100)
+                (ty::Tuple(fields), _) => {
+                    suggestions.extend(self.suggest_wrap_to_build_a_tuple( span, found, fields))
+                }
+                // If a byte was expected and the found expression is a char literal
+                // containing a single ASCII character, perhaps the user meant to write `b'c'` to
+                // specify a byte literal
+                (ty::Uint(ty::UintTy::U8), ty::Char) => {
+                    if let Ok(code) = self.tcx.sess().source_map().span_to_snippet(span)
+                        && let Some(code) = code.strip_prefix('\'').and_then(|s| s.strip_suffix('\''))
+                        && !code.starts_with("\\u") // forbid all Unicode escapes
+                        && code.chars().next().map_or(false, |c| c.is_ascii()) // forbids literal Unicode characters beyond ASCII
+                    {
+                        suggestions.push(TypeErrorAdditionalDiags::MeantByteLiteral { span, code: escape_literal(code) })
                     }
-                    escaped
                 }
-                let mut err = struct_span_err!(self.tcx.sess, span, E0308, "{}", failure_str);
-                if let Some((expected, found)) = trace.values.ty() {
-                    match (expected.kind(), found.kind()) {
-                        (ty::Tuple(_), ty::Tuple(_)) => {}
-                        // If a tuple of length one was expected and the found expression has
-                        // parentheses around it, perhaps the user meant to write `(expr,)` to
-                        // build a tuple (issue #86100)
-                        (ty::Tuple(fields), _) => {
-                            self.emit_tuple_wrap_err(&mut err, span, found, fields)
-                        }
-                        // If a byte was expected and the found expression is a char literal
-                        // containing a single ASCII character, perhaps the user meant to write `b'c'` to
-                        // specify a byte literal
-                        (ty::Uint(ty::UintTy::U8), ty::Char) => {
-                            if let Ok(code) = self.tcx.sess().source_map().span_to_snippet(span)
-                                && let Some(code) = code.strip_prefix('\'').and_then(|s| s.strip_suffix('\''))
-                                && !code.starts_with("\\u") // forbid all Unicode escapes
-                                && code.chars().next().map_or(false, |c| c.is_ascii()) // forbids literal Unicode characters beyond ASCII
-                            {
-                                err.span_suggestion(
-                                    span,
-                                    "if you meant to write a byte literal, prefix with `b`",
-                                    format!("b'{}'", escape_literal(code)),
-                                    Applicability::MachineApplicable,
-                                );
-                            }
-                        }
-                        // If a character was expected and the found expression is a string literal
-                        // containing a single character, perhaps the user meant to write `'c'` to
-                        // specify a character literal (issue #92479)
-                        (ty::Char, ty::Ref(_, r, _)) if r.is_str() => {
-                            if let Ok(code) = self.tcx.sess().source_map().span_to_snippet(span)
-                                && let Some(code) = code.strip_prefix('"').and_then(|s| s.strip_suffix('"'))
-                                && code.chars().count() == 1
-                            {
-                                err.span_suggestion(
-                                    span,
-                                    "if you meant to write a `char` literal, use single quotes",
-                                    format!("'{}'", escape_literal(code)),
-                                    Applicability::MachineApplicable,
-                                );
-                            }
-                        }
-                        // If a string was expected and the found expression is a character literal,
-                        // perhaps the user meant to write `"s"` to specify a string literal.
-                        (ty::Ref(_, r, _), ty::Char) if r.is_str() => {
-                            if let Ok(code) = self.tcx.sess().source_map().span_to_snippet(span) {
-                                if let Some(code) =
-                                    code.strip_prefix('\'').and_then(|s| s.strip_suffix('\''))
-                                {
-                                    err.span_suggestion(
-                                        span,
-                                        "if you meant to write a `str` literal, use double quotes",
-                                        format!("\"{}\"", escape_literal(code)),
-                                        Applicability::MachineApplicable,
-                                    );
-                                }
-                            }
-                        }
-                        // For code `if Some(..) = expr `, the type mismatch may be expected `bool` but found `()`,
-                        // we try to suggest to add the missing `let` for `if let Some(..) = expr`
-                        (ty::Bool, ty::Tuple(list)) => if list.len() == 0 {
-                            self.suggest_let_for_letchains(&mut err, &trace.cause, span);
-                        }
-                        (ty::Array(_, _), ty::Array(_, _)) => 'block: {
-                            let hir = self.tcx.hir();
-                            let TypeError::FixedArraySize(sz) = terr else {
-                                break 'block;
-                            };
-                            let tykind = match hir.find_by_def_id(trace.cause.body_id) {
-                                Some(hir::Node::Item(hir::Item {
-                                    kind: hir::ItemKind::Fn(_, _, body_id),
-                                    ..
-                                })) => {
-                                    let body = hir.body(*body_id);
-                                    struct LetVisitor<'v> {
-                                        span: Span,
-                                        result: Option<&'v hir::Ty<'v>>,
-                                    }
-                                    impl<'v> Visitor<'v> for LetVisitor<'v> {
-                                        fn visit_stmt(&mut self, s: &'v hir::Stmt<'v>) {
-                                            if self.result.is_some() {
-                                                return;
-                                            }
-                                            // Find a local statement where the initializer has
-                                            // the same span as the error and the type is specified.
-                                            if let hir::Stmt {
-                                                kind: hir::StmtKind::Local(hir::Local {
-                                                    init: Some(hir::Expr {
-                                                        span: init_span,
-                                                        ..
-                                                    }),
-                                                    ty: Some(array_ty),
-                                                    ..
-                                                }),
-                                                ..
-                                            } = s
-                                            && init_span == &self.span {
-                                                self.result = Some(*array_ty);
-                                            }
-                                        }
-                                    }
-                                    let mut visitor = LetVisitor {span, result: None};
-                                    visitor.visit_body(body);
-                                    visitor.result.map(|r| &r.peel_refs().kind)
-                                }
-                                Some(hir::Node::Item(hir::Item {
-                                    kind: hir::ItemKind::Const(ty, _),
-                                    ..
-                                })) => {
-                                    Some(&ty.peel_refs().kind)
-                                }
-                                _ => None
-                            };
-
-                            if let Some(tykind) = tykind
-                                && let hir::TyKind::Array(_, length) = tykind
-                                && let hir::ArrayLen::Body(hir::AnonConst { hir_id, .. }) = length
-                                && let Some(span) = self.tcx.hir().opt_span(*hir_id)
-                            {
-                                err.span_suggestion(
-                                    span,
-                                    "consider specifying the actual array length",
-                                    sz.found,
-                                    Applicability::MaybeIncorrect,
-                                );
-                            }
+                // If a character was expected and the found expression is a string literal
+                // containing a single character, perhaps the user meant to write `'c'` to
+                // specify a character literal (issue #92479)
+                (ty::Char, ty::Ref(_, r, _)) if r.is_str() => {
+                    if let Ok(code) = self.tcx.sess().source_map().span_to_snippet(span)
+                        && let Some(code) = code.strip_prefix('"').and_then(|s| s.strip_suffix('"'))
+                        && code.chars().count() == 1
+                    {
+                        suggestions.push(TypeErrorAdditionalDiags::MeantCharLiteral { span, code: escape_literal(code) })
+                    }
+                }
+                // If a string was expected and the found expression is a character literal,
+                // perhaps the user meant to write `"s"` to specify a string literal.
+                (ty::Ref(_, r, _), ty::Char) if r.is_str() => {
+                    if let Ok(code) = self.tcx.sess().source_map().span_to_snippet(span) {
+                        if let Some(code) =
+                            code.strip_prefix('\'').and_then(|s| s.strip_suffix('\''))
+                        {
+                            suggestions.push(TypeErrorAdditionalDiags::MeantStrLiteral { span, code: escape_literal(code) })
                         }
-                        _ => {}
                     }
                 }
-                let code = trace.cause.code();
-                if let &MatchExpressionArm(box MatchExpressionArmCause { source, .. }) = code
+                // For code `if Some(..) = expr `, the type mismatch may be expected `bool` but found `()`,
+                // we try to suggest to add the missing `let` for `if let Some(..) = expr`
+                (ty::Bool, ty::Tuple(list)) => if list.len() == 0 {
+                    suggestions.extend(self.suggest_let_for_letchains(&trace.cause, span));
+                }
+                (ty::Array(_, _), ty::Array(_, _)) => suggestions.extend(self.suggest_specify_actual_length(terr, trace, span)),
+                _ => {}
+            }
+        }
+        let code = trace.cause.code();
+        if let &MatchExpressionArm(box MatchExpressionArmCause { source, .. }) = code
                     && let hir::MatchSource::TryDesugar = source
                     && let Some((expected_ty, found_ty, _, _)) = self.values_str(trace.values)
                 {
-                    err.note(&format!(
-                        "`?` operator cannot convert from `{}` to `{}`",
-                        found_ty.content(),
-                        expected_ty.content(),
-                    ));
+                    suggestions.push(TypeErrorAdditionalDiags::TryCannotConvert { found: found_ty.content(), expected: expected_ty.content() });
                 }
-                err
+        suggestions
+    }
+
+    fn suggest_specify_actual_length(
+        &self,
+        terr: TypeError<'_>,
+        trace: &TypeTrace<'_>,
+        span: Span,
+    ) -> Option<TypeErrorAdditionalDiags> {
+        let hir = self.tcx.hir();
+        let TypeError::FixedArraySize(sz) = terr else {
+            return None;
+        };
+        let tykind = match hir.find_by_def_id(trace.cause.body_id) {
+            Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, _, body_id), .. })) => {
+                let body = hir.body(*body_id);
+                struct LetVisitor<'v> {
+                    span: Span,
+                    result: Option<&'v hir::Ty<'v>>,
+                }
+                impl<'v> Visitor<'v> for LetVisitor<'v> {
+                    fn visit_stmt(&mut self, s: &'v hir::Stmt<'v>) {
+                        if self.result.is_some() {
+                            return;
+                        }
+                        // Find a local statement where the initializer has
+                        // the same span as the error and the type is specified.
+                        if let hir::Stmt {
+                            kind: hir::StmtKind::Local(hir::Local {
+                                init: Some(hir::Expr {
+                                    span: init_span,
+                                    ..
+                                }),
+                                ty: Some(array_ty),
+                                ..
+                            }),
+                            ..
+                        } = s
+                        && init_span == &self.span {
+                            self.result = Some(*array_ty);
+                        }
+                    }
+                }
+                let mut visitor = LetVisitor { span, result: None };
+                visitor.visit_body(body);
+                visitor.result.map(|r| &r.peel_refs().kind)
             }
-            FailureCode::Error0644(failure_str) => {
-                struct_span_err!(self.tcx.sess, span, E0644, "{}", failure_str)
+            Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Const(ty, _), .. })) => {
+                Some(&ty.peel_refs().kind)
             }
+            _ => None,
         };
+        if let Some(tykind) = tykind
+            && let hir::TyKind::Array(_, length) = tykind
+            && let hir::ArrayLen::Body(hir::AnonConst { hir_id, .. }) = length
+            && let Some(span) = self.tcx.hir().opt_span(*hir_id)
+        {
+            Some(TypeErrorAdditionalDiags::ConsiderSpecifyingLength { span, length: sz.found })
+        } else {
+            None
+        }
+    }
+
+    pub fn report_and_explain_type_error(
+        &self,
+        trace: TypeTrace<'tcx>,
+        terr: TypeError<'tcx>,
+    ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
+        debug!("report_and_explain_type_error(trace={:?}, terr={:?})", trace, terr);
+
+        let span = trace.cause.span();
+        let failure_code = trace.cause.as_failure_code_diag(
+            terr,
+            span,
+            self.type_error_additional_suggestions(&trace, terr),
+        );
+        let mut diag = self.tcx.sess.create_err(failure_code);
         self.note_type_err(&mut diag, &trace.cause, None, Some(trace.values), terr, false, false);
         diag
     }
 
-    fn emit_tuple_wrap_err(
+    fn suggest_wrap_to_build_a_tuple(
         &self,
-        err: &mut Diagnostic,
         span: Span,
         found: Ty<'tcx>,
         expected_fields: &List<Ty<'tcx>>,
-    ) {
-        let [expected_tup_elem] = expected_fields[..] else { return };
+    ) -> Option<TypeErrorAdditionalDiags> {
+        let [expected_tup_elem] = expected_fields[..] else { return None};
 
         if !self.same_type_modulo_infer(expected_tup_elem, found) {
-            return;
+            return None;
         }
 
         let Ok(code) = self.tcx.sess().source_map().span_to_snippet(span)
-            else { return };
+            else { return None };
 
-        let msg = "use a trailing comma to create a tuple with one element";
-        if code.starts_with('(') && code.ends_with(')') {
+        let sugg = if code.starts_with('(') && code.ends_with(')') {
             let before_close = span.hi() - BytePos::from_u32(1);
-            err.span_suggestion(
-                span.with_hi(before_close).shrink_to_hi(),
-                msg,
-                ",",
-                Applicability::MachineApplicable,
-            );
+            TypeErrorAdditionalDiags::TupleOnlyComma {
+                span: span.with_hi(before_close).shrink_to_hi(),
+            }
         } else {
-            err.multipart_suggestion(
-                msg,
-                vec![(span.shrink_to_lo(), "(".into()), (span.shrink_to_hi(), ",)".into())],
-                Applicability::MachineApplicable,
-            );
-        }
+            TypeErrorAdditionalDiags::TupleAlsoParentheses {
+                span_low: span.shrink_to_lo(),
+                span_high: span.shrink_to_hi(),
+            }
+        };
+        Some(sugg)
     }
 
     fn values_str(
@@ -2124,6 +2122,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         match values {
             infer::Regions(exp_found) => self.expected_found_str(exp_found),
             infer::Terms(exp_found) => self.expected_found_str_term(exp_found),
+            infer::Aliases(exp_found) => self.expected_found_str(exp_found),
             infer::TraitRefs(exp_found) => {
                 let pretty_exp_found = ty::error::ExpectedFound {
                     expected: exp_found.expected.print_only_trait_path(),
@@ -2386,10 +2385,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 let suggestion =
                     if has_lifetimes { format!(" + {}", sub) } else { format!(": {}", sub) };
                 let mut suggestions = vec![(sp, suggestion)];
-                for add_lt_sugg in add_lt_suggs {
-                    if let Some(add_lt_sugg) = add_lt_sugg {
-                        suggestions.push(add_lt_sugg);
-                    }
+                for add_lt_sugg in add_lt_suggs.into_iter().flatten() {
+                    suggestions.push(add_lt_sugg);
                 }
                 err.multipart_suggestion_verbose(
                     format!("{msg}..."),
@@ -2413,11 +2410,9 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     };
                     let mut sugg =
                         vec![(sp, suggestion), (span.shrink_to_hi(), format!(" + {}", new_lt))];
-                    for add_lt_sugg in add_lt_suggs.clone() {
-                        if let Some(lt) = add_lt_sugg {
-                            sugg.push(lt);
-                            sugg.rotate_right(1);
-                        }
+                    for lt in add_lt_suggs.clone().into_iter().flatten() {
+                        sugg.push(lt);
+                        sugg.rotate_right(1);
                     }
                     // `MaybeIncorrect` due to issue #41966.
                     err.multipart_suggestion(msg, sugg, Applicability::MaybeIncorrect);
@@ -2688,11 +2683,6 @@ impl<'tcx> TypeRelation<'tcx> for SameTypeModuloInfer<'_, 'tcx> {
         self.0.tcx
     }
 
-    fn intercrate(&self) -> bool {
-        assert!(!self.0.intercrate);
-        false
-    }
-
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
         // Unused, only for consts which we treat as always equal
         ty::ParamEnv::empty()
@@ -2706,10 +2696,6 @@ impl<'tcx> TypeRelation<'tcx> for SameTypeModuloInfer<'_, 'tcx> {
         true
     }
 
-    fn mark_ambiguous(&mut self) {
-        bug!()
-    }
-
     fn relate_with_variance<T: relate::Relate<'tcx>>(
         &mut self,
         _variance: ty::Variance,
@@ -2828,15 +2814,21 @@ impl<'tcx> InferCtxt<'tcx> {
 }
 
 pub enum FailureCode {
-    Error0038(DefId),
-    Error0317(&'static str),
-    Error0580(&'static str),
-    Error0308(&'static str),
-    Error0644(&'static str),
+    Error0317,
+    Error0580,
+    Error0308,
+    Error0644,
 }
 
 pub trait ObligationCauseExt<'tcx> {
     fn as_failure_code(&self, terr: TypeError<'tcx>) -> FailureCode;
+
+    fn as_failure_code_diag(
+        &self,
+        terr: TypeError<'tcx>,
+        span: Span,
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    ) -> ObligationCauseFailureCode;
     fn as_requirement_str(&self) -> &'static str;
 }
 
@@ -2845,40 +2837,68 @@ impl<'tcx> ObligationCauseExt<'tcx> for ObligationCause<'tcx> {
         use self::FailureCode::*;
         use crate::traits::ObligationCauseCode::*;
         match self.code() {
+            IfExpressionWithNoElse => Error0317,
+            MainFunctionType => Error0580,
+            CompareImplItemObligation { .. }
+            | MatchExpressionArm(_)
+            | IfExpression { .. }
+            | LetElse
+            | StartFunctionType
+            | IntrinsicType
+            | MethodReceiver => Error0308,
+
+            // In the case where we have no more specific thing to
+            // say, also take a look at the error code, maybe we can
+            // tailor to that.
+            _ => match terr {
+                TypeError::CyclicTy(ty) if ty.is_closure() || ty.is_generator() => Error0644,
+                TypeError::IntrinsicCast => Error0308,
+                _ => Error0308,
+            },
+        }
+    }
+    fn as_failure_code_diag(
+        &self,
+        terr: TypeError<'tcx>,
+        span: Span,
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    ) -> ObligationCauseFailureCode {
+        use crate::traits::ObligationCauseCode::*;
+        match self.code() {
             CompareImplItemObligation { kind: ty::AssocKind::Fn, .. } => {
-                Error0308("method not compatible with trait")
+                ObligationCauseFailureCode::MethodCompat { span, subdiags }
             }
             CompareImplItemObligation { kind: ty::AssocKind::Type, .. } => {
-                Error0308("type not compatible with trait")
+                ObligationCauseFailureCode::TypeCompat { span, subdiags }
             }
             CompareImplItemObligation { kind: ty::AssocKind::Const, .. } => {
-                Error0308("const not compatible with trait")
-            }
-            MatchExpressionArm(box MatchExpressionArmCause { source, .. }) => {
-                Error0308(match source {
-                    hir::MatchSource::TryDesugar => "`?` operator has incompatible types",
-                    _ => "`match` arms have incompatible types",
-                })
-            }
-            IfExpression { .. } => Error0308("`if` and `else` have incompatible types"),
-            IfExpressionWithNoElse => Error0317("`if` may be missing an `else` clause"),
-            LetElse => Error0308("`else` clause of `let...else` does not diverge"),
-            MainFunctionType => Error0580("`main` function has wrong type"),
-            StartFunctionType => Error0308("`#[start]` function has wrong type"),
-            IntrinsicType => Error0308("intrinsic has wrong type"),
-            MethodReceiver => Error0308("mismatched `self` parameter type"),
+                ObligationCauseFailureCode::ConstCompat { span, subdiags }
+            }
+            MatchExpressionArm(box MatchExpressionArmCause { source, .. }) => match source {
+                hir::MatchSource::TryDesugar => {
+                    ObligationCauseFailureCode::TryCompat { span, subdiags }
+                }
+                _ => ObligationCauseFailureCode::MatchCompat { span, subdiags },
+            },
+            IfExpression { .. } => ObligationCauseFailureCode::IfElseDifferent { span, subdiags },
+            IfExpressionWithNoElse => ObligationCauseFailureCode::NoElse { span },
+            LetElse => ObligationCauseFailureCode::NoDiverge { span, subdiags },
+            MainFunctionType => ObligationCauseFailureCode::FnMainCorrectType { span },
+            StartFunctionType => ObligationCauseFailureCode::FnStartCorrectType { span, subdiags },
+            IntrinsicType => ObligationCauseFailureCode::IntristicCorrectType { span, subdiags },
+            MethodReceiver => ObligationCauseFailureCode::MethodCorrectType { span, subdiags },
 
             // In the case where we have no more specific thing to
             // say, also take a look at the error code, maybe we can
             // tailor to that.
             _ => match terr {
                 TypeError::CyclicTy(ty) if ty.is_closure() || ty.is_generator() => {
-                    Error0644("closure/generator type that references itself")
+                    ObligationCauseFailureCode::ClosureSelfref { span }
                 }
                 TypeError::IntrinsicCast => {
-                    Error0308("cannot coerce intrinsics to function pointers")
+                    ObligationCauseFailureCode::CantCoerce { span, subdiags }
                 }
-                _ => Error0308("mismatched types"),
+                _ => ObligationCauseFailureCode::Generic { span, subdiags },
             },
         }
     }
diff --git a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
index e242900fd..75cc4e257 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
@@ -1,5 +1,5 @@
 use crate::errors::{
-    AmbigousImpl, AmbigousReturn, AnnotationRequired, InferenceBadError, NeedTypeInfoInGenerator,
+    AmbigousReturn, AmbiguousImpl, AnnotationRequired, InferenceBadError, NeedTypeInfoInGenerator,
     SourceKindMultiSuggestion, SourceKindSubdiag,
 };
 use crate::infer::error_reporting::TypeErrCtxt;
@@ -10,14 +10,14 @@ use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, IntoDiagnosticArg};
 use rustc_hir as hir;
 use rustc_hir::def::Res;
 use rustc_hir::def::{CtorOf, DefKind, Namespace};
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{Body, Closure, Expr, ExprKind, FnRetTy, HirId, Local, LocalSource};
 use rustc_middle::hir::nested_filter;
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, AutoBorrow};
 use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter, Print, Printer};
-use rustc_middle::ty::{self, DefIdTree, InferConst};
+use rustc_middle::ty::{self, InferConst};
 use rustc_middle::ty::{GenericArg, GenericArgKind, SubstsRef};
 use rustc_middle::ty::{IsSuggestable, Ty, TyCtxt, TypeckResults};
 use rustc_span::symbol::{kw, sym, Ident};
@@ -358,7 +358,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 bad_label,
             }
             .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic),
-            TypeAnnotationNeeded::E0283 => AmbigousImpl {
+            TypeAnnotationNeeded::E0283 => AmbiguousImpl {
                 span,
                 source_kind,
                 source_name,
@@ -386,7 +386,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     #[instrument(level = "debug", skip(self, error_code))]
     pub fn emit_inference_failure_err(
         &self,
-        body_id: Option<hir::BodyId>,
+        body_def_id: LocalDefId,
         failure_span: Span,
         arg: GenericArg<'tcx>,
         error_code: TypeAnnotationNeeded,
@@ -403,8 +403,10 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         };
 
         let mut local_visitor = FindInferSourceVisitor::new(&self, typeck_results, arg);
-        if let Some(body_id) = body_id {
-            let expr = self.tcx.hir().expect_expr(body_id.hir_id);
+        if let Some(body_id) = self.tcx.hir().maybe_body_owned_by(
+            self.tcx.typeck_root_def_id(body_def_id.to_def_id()).expect_local(),
+        ) {
+            let expr = self.tcx.hir().body(body_id).value;
             local_visitor.visit_expr(expr);
         }
 
@@ -561,7 +563,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 bad_label: None,
             }
             .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic),
-            TypeAnnotationNeeded::E0283 => AmbigousImpl {
+            TypeAnnotationNeeded::E0283 => AmbiguousImpl {
                 span,
                 source_kind,
                 source_name: &name,
@@ -1189,11 +1191,14 @@ impl<'a, 'tcx> Visitor<'tcx> for FindInferSourceVisitor<'a, 'tcx> {
                 have_turbofish,
             } = args;
             let generics = tcx.generics_of(generics_def_id);
-            if let Some(argument_index) = generics
+            if let Some(mut argument_index) = generics
                 .own_substs(substs)
                 .iter()
                 .position(|&arg| self.generic_arg_contains_target(arg))
             {
+                if generics.parent.is_none() && generics.has_self {
+                    argument_index += 1;
+                }
                 let substs = self.infcx.resolve_vars_if_possible(substs);
                 let generic_args = &generics.own_substs_no_defaults(tcx, substs)
                     [generics.own_counts().lifetimes..];
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_relation.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_relation.rs
index e8d94f0c0..8a78a1956 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_relation.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_relation.rs
@@ -16,22 +16,34 @@ impl<'tcx> NiceRegionError<'_, 'tcx> {
         match &self.error {
             Some(RegionResolutionError::ConcreteFailure(
                 SubregionOrigin::RelateRegionParamBound(span),
-                Region(Interned(RePlaceholder(ty::Placeholder { name: sub_name, .. }), _)),
-                Region(Interned(RePlaceholder(ty::Placeholder { name: sup_name, .. }), _)),
+                Region(Interned(
+                    RePlaceholder(ty::Placeholder {
+                        bound: ty::BoundRegion { kind: sub_name, .. },
+                        ..
+                    }),
+                    _,
+                )),
+                Region(Interned(
+                    RePlaceholder(ty::Placeholder {
+                        bound: ty::BoundRegion { kind: sup_name, .. },
+                        ..
+                    }),
+                    _,
+                )),
             )) => {
                 let span = *span;
                 let (sub_span, sub_symbol) = match sub_name {
                     ty::BrNamed(def_id, symbol) => {
                         (Some(self.tcx().def_span(def_id)), Some(symbol))
                     }
-                    ty::BrAnon(_, span) => (*span, None),
+                    ty::BrAnon(span) => (*span, None),
                     ty::BrEnv => (None, None),
                 };
                 let (sup_span, sup_symbol) = match sup_name {
                     ty::BrNamed(def_id, symbol) => {
                         (Some(self.tcx().def_span(def_id)), Some(symbol))
                     }
-                    ty::BrAnon(_, span) => (*span, None),
+                    ty::BrAnon(span) => (*span, None),
                     ty::BrEnv => (None, None),
                 };
                 let diag = match (sub_span, sup_span, sub_symbol, sup_symbol) {
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
index b06ff10d8..22c1e3871 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
@@ -104,7 +104,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
         let (mention_influencer, influencer_point) =
             if sup_origin.span().overlaps(param.param_ty_span) {
                 // Account for `async fn` like in `async-await/issues/issue-62097.rs`.
-                // The desugaring of `async `fn`s causes `sup_origin` and `param` to point at the same
+                // The desugaring of `async fn`s causes `sup_origin` and `param` to point at the same
                 // place (but with different `ctxt`, hence `overlaps` instead of `==` above).
                 //
                 // This avoids the following:
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs
index db4b8af46..c5ef48fe3 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs
@@ -5,7 +5,7 @@ use crate::infer::error_reporting::nice_region_error::NiceRegionError;
 use crate::infer::TyCtxt;
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
-use rustc_middle::ty::{self, Binder, DefIdTree, Region, Ty, TypeVisitable};
+use rustc_middle::ty::{self, Binder, Region, Ty, TypeVisitable};
 use rustc_span::Span;
 
 /// Information about the anonymous region we are searching for.
diff --git a/compiler/rustc_infer/src/infer/error_reporting/note.rs b/compiler/rustc_infer/src/infer/error_reporting/note.rs
index 7ffe1fd20..07a9eff2d 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/note.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/note.rs
@@ -1,5 +1,5 @@
 use crate::errors::{
-    note_and_explain, FullfillReqLifetime, LfBoundNotSatisfied, OutlivesBound, OutlivesContent,
+    note_and_explain, FulfillReqLifetime, LfBoundNotSatisfied, OutlivesBound, OutlivesContent,
     RefLongerThanData, RegionOriginNote, WhereClauseSuggestions,
 };
 use crate::fluent_generated as fluent;
@@ -176,7 +176,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 let note = note_and_explain::RegionExplanation::new(
                     self.tcx, sub, opt_span, prefix, suffix,
                 );
-                FullfillReqLifetime { span, ty: self.resolve_vars_if_possible(ty), note }
+                FulfillReqLifetime { span, ty: self.resolve_vars_if_possible(ty), note }
                     .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
             }
             infer::RelateRegionParamBound(span) => {
@@ -306,9 +306,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             // Replace the explicit self type with `Self` for better suggestion rendering
             .with_self_ty(self.tcx, self.tcx.mk_ty_param(0, kw::SelfUpper))
             .substs;
-        let trait_item_substs =
-            ty::InternalSubsts::identity_for_item(self.tcx, impl_item_def_id.to_def_id())
-                .rebase_onto(self.tcx, impl_def_id, trait_substs);
+        let trait_item_substs = ty::InternalSubsts::identity_for_item(self.tcx, impl_item_def_id)
+            .rebase_onto(self.tcx, impl_def_id, trait_substs);
 
         let Ok(trait_predicates) = self
             .tcx
diff --git a/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs b/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs
index b33729d0b..b38bbdfe7 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs
@@ -1,7 +1,7 @@
 use super::TypeErrCtxt;
 use rustc_errors::Applicability::{MachineApplicable, MaybeIncorrect};
 use rustc_errors::{pluralize, Diagnostic, MultiSpan};
-use rustc_hir::{self as hir, def::DefKind};
+use rustc_hir as hir;
 use rustc_middle::traits::ObligationCauseCode;
 use rustc_middle::ty::error::ExpectedFound;
 use rustc_middle::ty::print::Printer;
@@ -75,7 +75,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                         diag.note("an associated type was expected, but a different one was found");
                     }
                     (ty::Param(p), ty::Alias(ty::Projection, proj)) | (ty::Alias(ty::Projection, proj), ty::Param(p))
-                        if tcx.def_kind(proj.def_id) != DefKind::ImplTraitPlaceholder =>
+                        if !tcx.is_impl_trait_in_trait(proj.def_id) =>
                     {
                         let p_def_id = tcx
                             .generics_of(body_owner_def_id)
@@ -222,7 +222,7 @@ impl<T> Trait<T> for X {
                             diag.span_label(p_span, "this type parameter");
                         }
                     }
-                    (ty::Alias(ty::Projection, proj_ty), _) if tcx.def_kind(proj_ty.def_id) != DefKind::ImplTraitPlaceholder => {
+                    (ty::Alias(ty::Projection, proj_ty), _) if !tcx.is_impl_trait_in_trait(proj_ty.def_id) => {
                         self.expected_projection(
                             diag,
                             proj_ty,
@@ -231,7 +231,7 @@ impl<T> Trait<T> for X {
                             cause.code(),
                         );
                     }
-                    (_, ty::Alias(ty::Projection, proj_ty)) if tcx.def_kind(proj_ty.def_id) != DefKind::ImplTraitPlaceholder => {
+                    (_, ty::Alias(ty::Projection, proj_ty)) if !tcx.is_impl_trait_in_trait(proj_ty.def_id) => {
                         let msg = format!(
                             "consider constraining the associated type `{}` to `{}`",
                             values.found, values.expected,
diff --git a/compiler/rustc_infer/src/infer/error_reporting/suggest.rs b/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
index 55dcfd05e..b5aeca12a 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
@@ -1,7 +1,7 @@
 use hir::def::CtorKind;
 use hir::intravisit::{walk_expr, walk_stmt, Visitor};
 use rustc_data_structures::fx::FxIndexSet;
-use rustc_errors::{Applicability, Diagnostic};
+use rustc_errors::Diagnostic;
 use rustc_hir as hir;
 use rustc_middle::traits::{
     IfExpressionCause, MatchExpressionArmCause, ObligationCause, ObligationCauseCode,
@@ -10,13 +10,23 @@ use rustc_middle::traits::{
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self as ty, IsSuggestable, Ty, TypeVisitableExt};
 use rustc_span::{sym, BytePos, Span};
+use rustc_target::abi::FieldIdx;
 
 use crate::errors::{
-    ConsiderAddingAwait, SuggAddLetForLetChains, SuggestRemoveSemiOrReturnBinding,
+    ConsiderAddingAwait, FnConsiderCasting, FnItemsAreDistinct, FnUniqTypes,
+    FunctionPointerSuggestion, SuggestAccessingField, SuggestAsRefWhereAppropriate,
+    SuggestBoxingForReturnImplTrait, SuggestRemoveSemiOrReturnBinding, SuggestTuplePatternMany,
+    SuggestTuplePatternOne, TypeErrorAdditionalDiags,
 };
 
 use super::TypeErrCtxt;
 
+#[derive(Clone, Copy)]
+pub enum SuggestAsRefKind {
+    Option,
+    Result,
+}
+
 impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     pub(super) fn suggest_remove_semi_or_return_binding(
         &self,
@@ -71,25 +81,20 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         return_sp: Span,
         arm_spans: impl Iterator<Item = Span>,
     ) {
-        err.multipart_suggestion(
-            "you could change the return type to be a boxed trait object",
-            vec![
-                (return_sp.with_hi(return_sp.lo() + BytePos(4)), "Box<dyn".to_string()),
-                (return_sp.shrink_to_hi(), ">".to_string()),
-            ],
-            Applicability::MaybeIncorrect,
-        );
-        let sugg = arm_spans
-            .flat_map(|sp| {
-                [(sp.shrink_to_lo(), "Box::new(".to_string()), (sp.shrink_to_hi(), ")".to_string())]
-                    .into_iter()
-            })
-            .collect::<Vec<_>>();
-        err.multipart_suggestion(
-            "if you change the return type to expect trait objects, box the returned expressions",
-            sugg,
-            Applicability::MaybeIncorrect,
-        );
+        let sugg = SuggestBoxingForReturnImplTrait::ChangeReturnType {
+            start_sp: return_sp.with_hi(return_sp.lo() + BytePos(4)),
+            end_sp: return_sp.shrink_to_hi(),
+        };
+        err.subdiagnostic(sugg);
+
+        let mut starts = Vec::new();
+        let mut ends = Vec::new();
+        for span in arm_spans {
+            starts.push(span.shrink_to_lo());
+            ends.push(span.shrink_to_hi());
+        }
+        let sugg = SuggestBoxingForReturnImplTrait::BoxReturnExpr { starts, ends };
+        err.subdiagnostic(sugg);
     }
 
     pub(super) fn suggest_tuple_pattern(
@@ -109,7 +114,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                         variant.fields.len() == 1 && variant.ctor_kind() == Some(CtorKind::Fn)
                     })
                     .filter_map(|variant| {
-                        let sole_field = &variant.fields[0];
+                        let sole_field = &variant.fields[FieldIdx::from_u32(0)];
                         let sole_field_ty = sole_field.ty(self.tcx, substs);
                         if self.same_type_modulo_infer(sole_field_ty, exp_found.found) {
                             let variant_path =
@@ -129,30 +134,21 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 match &compatible_variants[..] {
                     [] => {}
                     [variant] => {
-                        diag.multipart_suggestion_verbose(
-                            &format!("try wrapping the pattern in `{}`", variant),
-                            vec![
-                                (cause.span.shrink_to_lo(), format!("{}(", variant)),
-                                (cause.span.shrink_to_hi(), ")".to_string()),
-                            ],
-                            Applicability::MaybeIncorrect,
-                        );
+                        let sugg = SuggestTuplePatternOne {
+                            variant: variant.to_owned(),
+                            span_low: cause.span.shrink_to_lo(),
+                            span_high: cause.span.shrink_to_hi(),
+                        };
+                        diag.subdiagnostic(sugg);
                     }
                     _ => {
                         // More than one matching variant.
-                        diag.multipart_suggestions(
-                            &format!(
-                                "try wrapping the pattern in a variant of `{}`",
-                                self.tcx.def_path_str(expected_adt.did())
-                            ),
-                            compatible_variants.into_iter().map(|variant| {
-                                vec![
-                                    (cause.span.shrink_to_lo(), format!("{}(", variant)),
-                                    (cause.span.shrink_to_hi(), ")".to_string()),
-                                ]
-                            }),
-                            Applicability::MaybeIncorrect,
-                        );
+                        let sugg = SuggestTuplePatternMany {
+                            path: self.tcx.def_path_str(expected_adt.did()),
+                            cause_span: cause.span,
+                            compatible_variants,
+                        };
+                        diag.subdiagnostic(sugg);
                     }
                 }
             }
@@ -255,15 +251,6 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         }
     }
 
-    pub fn suggest_await_on_future(&self, diag: &mut Diagnostic, sp: Span) {
-        diag.span_suggestion_verbose(
-            sp.shrink_to_hi(),
-            "consider `await`ing on the `Future`",
-            ".await",
-            Applicability::MaybeIncorrect,
-        );
-    }
-
     pub(super) fn suggest_accessing_field_where_appropriate(
         &self,
         cause: &ObligationCause<'tcx>,
@@ -290,21 +277,13 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 if let ObligationCauseCode::Pattern { span: Some(span), .. } = *cause.code() {
                     if let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(span) {
                         let suggestion = if expected_def.is_struct() {
-                            format!("{}.{}", snippet, name)
+                            SuggestAccessingField::Safe { span, snippet, name, ty }
                         } else if expected_def.is_union() {
-                            format!("unsafe {{ {}.{} }}", snippet, name)
+                            SuggestAccessingField::Unsafe { span, snippet, name, ty }
                         } else {
                             return;
                         };
-                        diag.span_suggestion(
-                            span,
-                            &format!(
-                                "you might have meant to use field `{}` whose type is `{}`",
-                                name, ty
-                            ),
-                            suggestion,
-                            Applicability::MaybeIncorrect,
-                        );
+                        diag.subdiagnostic(suggestion);
                     }
                 }
             }
@@ -320,15 +299,15 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         diag: &mut Diagnostic,
     ) {
         if let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(span)
-            && let Some(msg) = self.should_suggest_as_ref(exp_found.expected, exp_found.found)
+            && let Some(msg) = self.should_suggest_as_ref_kind(exp_found.expected, exp_found.found)
         {
-            diag.span_suggestion(
-                span,
-                msg,
-                // HACK: fix issue# 100605, suggesting convert from &Option<T> to Option<&T>, remove the extra `&`
-                format!("{}.as_ref()", snippet.trim_start_matches('&')),
-                Applicability::MachineApplicable,
-            );
+            // HACK: fix issue# 100605, suggesting convert from &Option<T> to Option<&T>, remove the extra `&`
+            let snippet = snippet.trim_start_matches('&');
+            let subdiag = match msg {
+                SuggestAsRefKind::Option => SuggestAsRefWhereAppropriate::Option { span, snippet },
+                SuggestAsRefKind::Result => SuggestAsRefWhereAppropriate::Result { span, snippet },
+            };
+            diag.subdiagnostic(subdiag);
         }
     }
 
@@ -356,36 +335,24 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
                 if !self.same_type_modulo_infer(*found_sig, *expected_sig)
                     || !sig.is_suggestable(self.tcx, true)
-                    || ty::util::is_intrinsic(self.tcx, *did)
+                    || self.tcx.is_intrinsic(*did)
                 {
                     return;
                 }
 
-                let (msg, sug) = match (expected.is_ref(), found.is_ref()) {
-                    (true, false) => {
-                        let msg = "consider using a reference";
-                        let sug = format!("&{fn_name}");
-                        (msg, sug)
-                    }
-                    (false, true) => {
-                        let msg = "consider removing the reference";
-                        let sug = format!("{fn_name}");
-                        (msg, sug)
-                    }
+                let sugg = match (expected.is_ref(), found.is_ref()) {
+                    (true, false) => FunctionPointerSuggestion::UseRef { span, fn_name },
+                    (false, true) => FunctionPointerSuggestion::RemoveRef { span, fn_name },
                     (true, true) => {
-                        diag.note("fn items are distinct from fn pointers");
-                        let msg = "consider casting to a fn pointer";
-                        let sug = format!("&({fn_name} as {sig})");
-                        (msg, sug)
+                        diag.subdiagnostic(FnItemsAreDistinct);
+                        FunctionPointerSuggestion::CastRef { span, fn_name, sig: *sig }
                     }
                     (false, false) => {
-                        diag.note("fn items are distinct from fn pointers");
-                        let msg = "consider casting to a fn pointer";
-                        let sug = format!("{fn_name} as {sig}");
-                        (msg, sug)
+                        diag.subdiagnostic(FnItemsAreDistinct);
+                        FunctionPointerSuggestion::Cast { span, fn_name, sig: *sig }
                     }
                 };
-                diag.span_suggestion_verbose(span, msg, sug, Applicability::MaybeIncorrect);
+                diag.subdiagnostic(sugg);
             }
             (ty::FnDef(did1, substs1), ty::FnDef(did2, substs2)) => {
                 let expected_sig =
@@ -394,30 +361,36 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     &(self.normalize_fn_sig)(self.tcx.fn_sig(*did2).subst(self.tcx, substs2));
 
                 if self.same_type_modulo_infer(*expected_sig, *found_sig) {
-                    diag.note("different fn items have unique types, even if their signatures are the same");
+                    diag.subdiagnostic(FnUniqTypes);
                 }
 
                 if !self.same_type_modulo_infer(*found_sig, *expected_sig)
                     || !found_sig.is_suggestable(self.tcx, true)
                     || !expected_sig.is_suggestable(self.tcx, true)
-                    || ty::util::is_intrinsic(self.tcx, *did1)
-                    || ty::util::is_intrinsic(self.tcx, *did2)
+                    || self.tcx.is_intrinsic(*did1)
+                    || self.tcx.is_intrinsic(*did2)
                 {
                     return;
                 }
 
                 let fn_name = self.tcx.def_path_str_with_substs(*did2, substs2);
                 let sug = if found.is_ref() {
-                    format!("&({fn_name} as {found_sig})")
+                    FunctionPointerSuggestion::CastBothRef {
+                        span,
+                        fn_name,
+                        found_sig: *found_sig,
+                        expected_sig: *expected_sig,
+                    }
                 } else {
-                    format!("{fn_name} as {found_sig}")
+                    FunctionPointerSuggestion::CastBoth {
+                        span,
+                        fn_name,
+                        found_sig: *found_sig,
+                        expected_sig: *expected_sig,
+                    }
                 };
 
-                let msg = format!(
-                    "consider casting both fn items to fn pointers using `as {expected_sig}`"
-                );
-
-                diag.span_suggestion_hidden(span, msg, sug, Applicability::MaybeIncorrect);
+                diag.subdiagnostic(sug);
             }
             (ty::FnDef(did, substs), ty::FnPtr(sig)) => {
                 let expected_sig =
@@ -436,7 +409,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     format!("{fn_name} as {found_sig}")
                 };
 
-                diag.help(&format!("consider casting the fn item to a fn pointer: `{}`", casting));
+                diag.subdiagnostic(FnConsiderCasting { casting });
             }
             _ => {
                 return;
@@ -444,23 +417,19 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         };
     }
 
-    pub fn should_suggest_as_ref(&self, expected: Ty<'tcx>, found: Ty<'tcx>) -> Option<&str> {
+    pub fn should_suggest_as_ref_kind(
+        &self,
+        expected: Ty<'tcx>,
+        found: Ty<'tcx>,
+    ) -> Option<SuggestAsRefKind> {
         if let (ty::Adt(exp_def, exp_substs), ty::Ref(_, found_ty, _)) =
             (expected.kind(), found.kind())
         {
             if let ty::Adt(found_def, found_substs) = *found_ty.kind() {
                 if exp_def == &found_def {
                     let have_as_ref = &[
-                        (
-                            sym::Option,
-                            "you can convert from `&Option<T>` to `Option<&T>` using \
-                        `.as_ref()`",
-                        ),
-                        (
-                            sym::Result,
-                            "you can convert from `&Result<T, E>` to \
-                        `Result<&T, &E>` using `.as_ref()`",
-                        ),
+                        (sym::Option, SuggestAsRefKind::Option),
+                        (sym::Result, SuggestAsRefKind::Result),
                     ];
                     if let Some(msg) = have_as_ref.iter().find_map(|(name, msg)| {
                         self.tcx.is_diagnostic_item(*name, exp_def.did()).then_some(msg)
@@ -494,15 +463,28 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         None
     }
 
+    // FIXME: Remove once `rustc_hir_typeck` is migrated to diagnostic structs
+    pub fn should_suggest_as_ref(&self, expected: Ty<'tcx>, found: Ty<'tcx>) -> Option<&str> {
+        match self.should_suggest_as_ref_kind(expected, found) {
+            Some(SuggestAsRefKind::Option) => Some(
+                "you can convert from `&Option<T>` to `Option<&T>` using \
+            `.as_ref()`",
+            ),
+            Some(SuggestAsRefKind::Result) => Some(
+                "you can convert from `&Result<T, E>` to \
+            `Result<&T, &E>` using `.as_ref()`",
+            ),
+            None => None,
+        }
+    }
     /// Try to find code with pattern `if Some(..) = expr`
     /// use a `visitor` to mark the `if` which its span contains given error span,
     /// and then try to find a assignment in the `cond` part, which span is equal with error span
     pub(super) fn suggest_let_for_letchains(
         &self,
-        err: &mut Diagnostic,
         cause: &ObligationCause<'_>,
         span: Span,
-    ) {
+    ) -> Option<TypeErrorAdditionalDiags> {
         let hir = self.tcx.hir();
         if let Some(node) = self.tcx.hir().find_by_def_id(cause.body_id) &&
             let hir::Node::Item(hir::Item {
@@ -549,9 +531,10 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         let mut visitor = IfVisitor { err_span: span, found_if: false, result: false };
         visitor.visit_body(&body);
         if visitor.result {
-                err.subdiagnostic(SuggAddLetForLetChains{span: span.shrink_to_lo()});
+                return Some(TypeErrorAdditionalDiags::AddLetForLetChains{span: span.shrink_to_lo()});
             }
         }
+        None
     }
 }
 
diff --git a/compiler/rustc_infer/src/infer/freshen.rs b/compiler/rustc_infer/src/infer/freshen.rs
index f09f93abf..d89f63e5c 100644
--- a/compiler/rustc_infer/src/infer/freshen.rs
+++ b/compiler/rustc_infer/src/infer/freshen.rs
@@ -43,18 +43,16 @@ pub struct TypeFreshener<'a, 'tcx> {
     const_freshen_count: u32,
     ty_freshen_map: FxHashMap<ty::InferTy, Ty<'tcx>>,
     const_freshen_map: FxHashMap<ty::InferConst<'tcx>, ty::Const<'tcx>>,
-    keep_static: bool,
 }
 
 impl<'a, 'tcx> TypeFreshener<'a, 'tcx> {
-    pub fn new(infcx: &'a InferCtxt<'tcx>, keep_static: bool) -> TypeFreshener<'a, 'tcx> {
+    pub fn new(infcx: &'a InferCtxt<'tcx>) -> TypeFreshener<'a, 'tcx> {
         TypeFreshener {
             infcx,
             ty_freshen_count: 0,
             const_freshen_count: 0,
             ty_freshen_map: Default::default(),
             const_freshen_map: Default::default(),
-            keep_static,
         }
     }
 
@@ -121,18 +119,9 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for TypeFreshener<'a, 'tcx> {
             | ty::ReFree(_)
             | ty::ReVar(_)
             | ty::RePlaceholder(..)
+            | ty::ReStatic
             | ty::ReError(_)
-            | ty::ReErased => {
-                // replace all free regions with 'erased
-                self.interner().lifetimes.re_erased
-            }
-            ty::ReStatic => {
-                if self.keep_static {
-                    r
-                } else {
-                    self.interner().lifetimes.re_erased
-                }
-            }
+            | ty::ReErased => self.interner().lifetimes.re_erased,
         }
     }
 
diff --git a/compiler/rustc_infer/src/infer/glb.rs b/compiler/rustc_infer/src/infer/glb.rs
index 49df393d8..2f659d9a6 100644
--- a/compiler/rustc_infer/src/infer/glb.rs
+++ b/compiler/rustc_infer/src/infer/glb.rs
@@ -2,8 +2,8 @@
 
 use super::combine::{CombineFields, ObligationEmittingRelation};
 use super::lattice::{self, LatticeDir};
-use super::InferCtxt;
 use super::Subtype;
+use super::{DefineOpaqueTypes, InferCtxt};
 
 use crate::traits::{ObligationCause, PredicateObligations};
 use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation};
@@ -29,11 +29,6 @@ impl<'tcx> TypeRelation<'tcx> for Glb<'_, '_, 'tcx> {
         "Glb"
     }
 
-    fn intercrate(&self) -> bool {
-        assert!(!self.fields.infcx.intercrate);
-        false
-    }
-
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.fields.tcx()
     }
@@ -46,10 +41,6 @@ impl<'tcx> TypeRelation<'tcx> for Glb<'_, '_, 'tcx> {
         self.a_is_expected
     }
 
-    fn mark_ambiguous(&mut self) {
-        bug!("mark_ambiguous used outside of coherence");
-    }
-
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
         variance: ty::Variance,
@@ -142,7 +133,7 @@ impl<'combine, 'infcx, 'tcx> LatticeDir<'infcx, 'tcx> for Glb<'combine, 'infcx,
         Ok(())
     }
 
-    fn define_opaque_types(&self) -> bool {
+    fn define_opaque_types(&self) -> DefineOpaqueTypes {
         self.fields.define_opaque_types
     }
 }
@@ -155,4 +146,9 @@ impl<'tcx> ObligationEmittingRelation<'tcx> for Glb<'_, '_, 'tcx> {
     fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
         self.fields.register_obligations(obligations);
     }
+
+    fn alias_relate_direction(&self) -> ty::AliasRelationDirection {
+        // FIXME(deferred_projection_equality): This isn't right, I think?
+        ty::AliasRelationDirection::Equate
+    }
 }
diff --git a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs b/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
index d1897cf24..a63cfbc91 100644
--- a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
+++ b/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
@@ -82,20 +82,20 @@ impl<'tcx> InferCtxt<'tcx> {
 
         let delegate = FnMutDelegate {
             regions: &mut |br: ty::BoundRegion| {
-                self.tcx.mk_re_placeholder(ty::PlaceholderRegion {
-                    universe: next_universe,
-                    name: br.kind,
-                })
+                self.tcx
+                    .mk_re_placeholder(ty::PlaceholderRegion { universe: next_universe, bound: br })
             },
             types: &mut |bound_ty: ty::BoundTy| {
                 self.tcx.mk_placeholder(ty::PlaceholderType {
                     universe: next_universe,
-                    name: bound_ty.kind,
+                    bound: bound_ty,
                 })
             },
             consts: &mut |bound_var: ty::BoundVar, ty| {
-                self.tcx
-                    .mk_const(ty::PlaceholderConst { universe: next_universe, name: bound_var }, ty)
+                self.tcx.mk_const(
+                    ty::PlaceholderConst { universe: next_universe, bound: bound_var },
+                    ty,
+                )
             },
         };
 
diff --git a/compiler/rustc_infer/src/infer/lattice.rs b/compiler/rustc_infer/src/infer/lattice.rs
index f377ac1d1..7f4c141b9 100644
--- a/compiler/rustc_infer/src/infer/lattice.rs
+++ b/compiler/rustc_infer/src/infer/lattice.rs
@@ -19,7 +19,7 @@
 
 use super::combine::ObligationEmittingRelation;
 use super::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use super::InferCtxt;
+use super::{DefineOpaqueTypes, InferCtxt};
 
 use crate::traits::ObligationCause;
 use rustc_middle::ty::relate::RelateResult;
@@ -36,7 +36,7 @@ pub trait LatticeDir<'f, 'tcx>: ObligationEmittingRelation<'tcx> {
 
     fn cause(&self) -> &ObligationCause<'tcx>;
 
-    fn define_opaque_types(&self) -> bool;
+    fn define_opaque_types(&self) -> DefineOpaqueTypes;
 
     // Relates the type `v` to `a` and `b` such that `v` represents
     // the LUB/GLB of `a` and `b` as appropriate.
@@ -110,7 +110,7 @@ where
         ) if a_def_id == b_def_id => infcx.super_combine_tys(this, a, b),
         (&ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }), _)
         | (_, &ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }))
-            if this.define_opaque_types() && def_id.is_local() =>
+            if this.define_opaque_types() == DefineOpaqueTypes::Yes && def_id.is_local() =>
         {
             this.register_obligations(
                 infcx
diff --git a/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs b/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs
index 2c4803550..f298b95ca 100644
--- a/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs
+++ b/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs
@@ -13,7 +13,7 @@ use rustc_data_structures::graph::implementation::{
     Direction, Graph, NodeIndex, INCOMING, OUTGOING,
 };
 use rustc_data_structures::intern::Interned;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::PlaceholderRegion;
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -132,7 +132,6 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
         }
 
         let graph = self.construct_graph();
-        self.expand_givens(&graph);
         self.expansion(&mut var_data);
         self.collect_errors(&mut var_data, errors);
         self.collect_var_errors(&var_data, &graph, errors);
@@ -164,38 +163,6 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
         }
     }
 
-    fn expand_givens(&mut self, graph: &RegionGraph<'_>) {
-        // Givens are a kind of horrible hack to account for
-        // constraints like 'c <= '0 that are known to hold due to
-        // closure signatures (see the comment above on the `givens`
-        // field). They should go away. But until they do, the role
-        // of this fn is to account for the transitive nature:
-        //
-        //     Given 'c <= '0
-        //     and   '0 <= '1
-        //     then  'c <= '1
-
-        let seeds: Vec<_> = self.data.givens.iter().cloned().collect();
-        for (r, vid) in seeds {
-            // While all things transitively reachable in the graph
-            // from the variable (`'0` in the example above).
-            let seed_index = NodeIndex(vid.index() as usize);
-            for succ_index in graph.depth_traverse(seed_index, OUTGOING) {
-                let succ_index = succ_index.0;
-
-                // The first N nodes correspond to the region
-                // variables. Other nodes correspond to constant
-                // regions.
-                if succ_index < self.num_vars() {
-                    let succ_vid = RegionVid::new(succ_index);
-
-                    // Add `'c <= '1`.
-                    self.data.givens.insert((r, succ_vid));
-                }
-            }
-        }
-    }
-
     /// Gets the LUb of a given region and the empty region
     fn lub_empty(&self, a_region: Region<'tcx>) -> Result<Region<'tcx>, PlaceholderRegion> {
         match *a_region {
@@ -236,7 +203,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
 
         // Tracks the `VarSubVar` constraints generated for each region vid. We
         // later use this to expand across vids.
-        let mut constraints = IndexVec::from_elem_n(Vec::new(), var_values.values.len());
+        let mut constraints = IndexVec::from_elem(Vec::new(), &var_values.values);
         // Tracks the changed region vids.
         let mut changes = Vec::new();
         for constraint in self.data.constraints.keys() {
@@ -362,18 +329,6 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
     ) -> bool {
         debug!("expand_node({:?}, {:?} == {:?})", a_region, b_vid, b_data);
 
-        match *a_region {
-            // Check if this relationship is implied by a given.
-            ty::ReEarlyBound(_) | ty::ReFree(_) => {
-                if self.data.givens.contains(&(a_region, b_vid)) {
-                    debug!("given");
-                    return false;
-                }
-            }
-
-            _ => {}
-        }
-
         match *b_data {
             VarValue::Empty(empty_ui) => {
                 let lub = match self.lub_empty(a_region) {
@@ -768,7 +723,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
     fn collect_error_for_expanding_node(
         &self,
         graph: &RegionGraph<'tcx>,
-        dup_vec: &mut IndexVec<RegionVid, Option<RegionVid>>,
+        dup_vec: &mut IndexSlice<RegionVid, Option<RegionVid>>,
         node_idx: RegionVid,
         errors: &mut Vec<RegionResolutionError<'tcx>>,
     ) {
@@ -891,7 +846,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
         graph: &RegionGraph<'tcx>,
         orig_node_idx: RegionVid,
         dir: Direction,
-        mut dup_vec: Option<&mut IndexVec<RegionVid, Option<RegionVid>>>,
+        mut dup_vec: Option<&mut IndexSlice<RegionVid, Option<RegionVid>>>,
     ) -> (Vec<RegionAndOrigin<'tcx>>, FxHashSet<RegionVid>, bool) {
         struct WalkState<'tcx> {
             set: FxHashSet<RegionVid>,
diff --git a/compiler/rustc_infer/src/infer/lub.rs b/compiler/rustc_infer/src/infer/lub.rs
index c871ccb21..e41ec7e6c 100644
--- a/compiler/rustc_infer/src/infer/lub.rs
+++ b/compiler/rustc_infer/src/infer/lub.rs
@@ -2,8 +2,8 @@
 
 use super::combine::{CombineFields, ObligationEmittingRelation};
 use super::lattice::{self, LatticeDir};
-use super::InferCtxt;
 use super::Subtype;
+use super::{DefineOpaqueTypes, InferCtxt};
 
 use crate::traits::{ObligationCause, PredicateObligations};
 use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation};
@@ -29,11 +29,6 @@ impl<'tcx> TypeRelation<'tcx> for Lub<'_, '_, 'tcx> {
         "Lub"
     }
 
-    fn intercrate(&self) -> bool {
-        assert!(!self.fields.infcx.intercrate);
-        false
-    }
-
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.fields.tcx()
     }
@@ -46,10 +41,6 @@ impl<'tcx> TypeRelation<'tcx> for Lub<'_, '_, 'tcx> {
         self.a_is_expected
     }
 
-    fn mark_ambiguous(&mut self) {
-        bug!("mark_ambiguous used outside of coherence");
-    }
-
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
         variance: ty::Variance,
@@ -142,7 +133,7 @@ impl<'combine, 'infcx, 'tcx> LatticeDir<'infcx, 'tcx> for Lub<'combine, 'infcx,
         Ok(())
     }
 
-    fn define_opaque_types(&self) -> bool {
+    fn define_opaque_types(&self) -> DefineOpaqueTypes {
         self.fields.define_opaque_types
     }
 }
@@ -155,4 +146,9 @@ impl<'tcx> ObligationEmittingRelation<'tcx> for Lub<'_, '_, 'tcx> {
     fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
         self.fields.register_obligations(obligations)
     }
+
+    fn alias_relate_direction(&self) -> ty::AliasRelationDirection {
+        // FIXME(deferred_projection_equality): This isn't right, I think?
+        ty::AliasRelationDirection::Equate
+    }
 }
diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs
index bd1f96635..66f51328b 100644
--- a/compiler/rustc_infer/src/infer/mod.rs
+++ b/compiler/rustc_infer/src/infer/mod.rs
@@ -1,3 +1,4 @@
+pub use self::at::DefineOpaqueTypes;
 pub use self::freshen::TypeFreshener;
 pub use self::lexical_region_resolve::RegionResolutionError;
 pub use self::LateBoundRegionConversionTime::*;
@@ -38,13 +39,13 @@ use rustc_span::Span;
 
 use std::cell::{Cell, RefCell};
 use std::fmt;
+use std::ops::Drop;
 
 use self::combine::CombineFields;
 use self::error_reporting::TypeErrCtxt;
 use self::free_regions::RegionRelations;
 use self::lexical_region_resolve::LexicalRegionResolutions;
-use self::outlives::env::OutlivesEnvironment;
-use self::region_constraints::{GenericKind, RegionConstraintData, VarInfos, VerifyBound};
+use self::region_constraints::{GenericKind, VarInfos, VerifyBound};
 use self::region_constraints::{
     RegionConstraintCollector, RegionConstraintStorage, RegionSnapshot,
 };
@@ -94,10 +95,10 @@ pub(crate) type UnificationTable<'a, 'tcx, T> = ut::UnificationTable<
 /// call to `start_snapshot` and `rollback_to`.
 #[derive(Clone)]
 pub struct InferCtxtInner<'tcx> {
-    /// Cache for projections. This cache is snapshotted along with the infcx.
+    /// Cache for projections.
     ///
-    /// Public so that `traits::project` can use it.
-    pub projection_cache: traits::ProjectionCacheStorage<'tcx>,
+    /// This cache is snapshotted along with the infcx.
+    projection_cache: traits::ProjectionCacheStorage<'tcx>,
 
     /// We instantiate `UnificationTable` with `bounds<Ty>` because the types
     /// that might instantiate a general type variable have an order,
@@ -114,24 +115,26 @@ pub struct InferCtxtInner<'tcx> {
     float_unification_storage: ut::UnificationTableStorage<ty::FloatVid>,
 
     /// Tracks the set of region variables and the constraints between them.
+    ///
     /// This is initially `Some(_)` but when
     /// `resolve_regions_and_report_errors` is invoked, this gets set to `None`
     /// -- further attempts to perform unification, etc., may fail if new
     /// region constraints would've been added.
     region_constraint_storage: Option<RegionConstraintStorage<'tcx>>,
 
-    /// A set of constraints that regionck must validate. Each
-    /// constraint has the form `T:'a`, meaning "some type `T` must
+    /// A set of constraints that regionck must validate.
+    ///
+    /// Each constraint has the form `T:'a`, meaning "some type `T` must
     /// outlive the lifetime 'a". These constraints derive from
     /// instantiated type parameters. So if you had a struct defined
-    /// like
+    /// like the following:
     /// ```ignore (illustrative)
-    ///     struct Foo<T:'static> { ... }
+    /// struct Foo<T: 'static> { ... }
     /// ```
-    /// then in some expression `let x = Foo { ... }` it will
+    /// In some expression `let x = Foo { ... }`, it will
     /// instantiate the type parameter `T` with a fresh type `$0`. At
     /// the same time, it will record a region obligation of
-    /// `$0:'static`. This will get checked later by regionck. (We
+    /// `$0: 'static`. This will get checked later by regionck. (We
     /// can't generally check these things right away because we have
     /// to wait until types are resolved.)
     ///
@@ -185,6 +188,16 @@ impl<'tcx> InferCtxtInner<'tcx> {
     }
 
     #[inline]
+    fn try_type_variables_probe_ref(
+        &self,
+        vid: ty::TyVid,
+    ) -> Option<&type_variable::TypeVariableValue<'tcx>> {
+        // Uses a read-only view of the unification table, this way we don't
+        // need an undo log.
+        self.type_variable_storage.eq_relations_ref().try_probe_value(vid)
+    }
+
+    #[inline]
     fn type_variables(&mut self) -> type_variable::TypeVariableTable<'_, 'tcx> {
         self.type_variable_storage.with_log(&mut self.undo_log)
     }
@@ -268,7 +281,7 @@ pub struct InferCtxt<'tcx> {
     /// Caches the results of trait evaluation.
     pub evaluation_cache: select::EvaluationCache<'tcx>,
 
-    /// the set of predicates on which errors have been reported, to
+    /// The set of predicates on which errors have been reported, to
     /// avoid reporting the same error twice.
     pub reported_trait_errors: RefCell<FxIndexMap<Span, Vec<ty::Predicate<'tcx>>>>,
 
@@ -291,7 +304,7 @@ pub struct InferCtxt<'tcx> {
     tainted_by_errors: Cell<Option<ErrorGuaranteed>>,
 
     /// Track how many errors were reported when this infcx is created.
-    /// If the number of errors increases, that's also a sign (line
+    /// If the number of errors increases, that's also a sign (like
     /// `tainted_by_errors`) to avoid reporting certain kinds of errors.
     // FIXME(matthewjasper) Merge into `tainted_by_errors`
     err_count_on_creation: usize,
@@ -313,7 +326,7 @@ pub struct InferCtxt<'tcx> {
     /// During coherence we have to assume that other crates may add
     /// additional impls which we currently don't know about.
     ///
-    /// To deal with this evaluation should be conservative
+    /// To deal with this evaluation, we should be conservative
     /// and consider the possibility of impls from outside this crate.
     /// This comes up primarily when resolving ambiguity. Imagine
     /// there is some trait reference `$0: Bar` where `$0` is an
@@ -323,12 +336,17 @@ pub struct InferCtxt<'tcx> {
     /// bound to some type that in a downstream crate that implements
     /// `Bar`.
     ///
-    /// Outside of coherence we set this to false because we are only
+    /// Outside of coherence, we set this to false because we are only
     /// interested in types that the user could actually have written.
     /// In other words, we consider `$0: Bar` to be unimplemented if
     /// there is no type that the user could *actually name* that
     /// would satisfy it. This avoids crippling inference, basically.
     pub intercrate: bool,
+
+    /// Flag that is set when we enter canonicalization. Used for debugging to ensure
+    /// that we only collect region information for `BorrowckInferCtxt::reg_var_to_origin`
+    /// inside non-canonicalization contexts.
+    inside_canonicalization_ctxt: Cell<bool>,
 }
 
 /// See the `error_reporting` module for more details.
@@ -336,6 +354,7 @@ pub struct InferCtxt<'tcx> {
 pub enum ValuePairs<'tcx> {
     Regions(ExpectedFound<ty::Region<'tcx>>),
     Terms(ExpectedFound<ty::Term<'tcx>>),
+    Aliases(ExpectedFound<ty::AliasTy<'tcx>>),
     TraitRefs(ExpectedFound<ty::TraitRef<'tcx>>),
     PolyTraitRefs(ExpectedFound<ty::PolyTraitRef<'tcx>>),
     Sigs(ExpectedFound<ty::FnSig<'tcx>>),
@@ -373,7 +392,7 @@ pub enum SubregionOrigin<'tcx> {
     Subtype(Box<TypeTrace<'tcx>>),
 
     /// When casting `&'a T` to an `&'b Trait` object,
-    /// relating `'a` to `'b`
+    /// relating `'a` to `'b`.
     RelateObjectBound(Span),
 
     /// Some type parameter was instantiated with the given type,
@@ -384,7 +403,7 @@ pub enum SubregionOrigin<'tcx> {
     /// that must outlive some other region.
     RelateRegionParamBound(Span),
 
-    /// Creating a pointer `b` to contents of another reference
+    /// Creating a pointer `b` to contents of another reference.
     Reborrow(Span),
 
     /// (&'a &'b T) where a >= b
@@ -398,7 +417,7 @@ pub enum SubregionOrigin<'tcx> {
         trait_item_def_id: DefId,
     },
 
-    /// Checking that the bounds of a trait's associated type hold for a given impl
+    /// Checking that the bounds of a trait's associated type hold for a given impl.
     CheckAssociatedTypeBounds {
         parent: Box<SubregionOrigin<'tcx>>,
         impl_item_def_id: LocalDefId,
@@ -435,32 +454,33 @@ pub enum LateBoundRegionConversionTime {
     AssocTypeProjection(DefId),
 }
 
-/// Reasons to create a region inference variable
+/// Reasons to create a region inference variable.
 ///
-/// See `error_reporting` module for more details
+/// See `error_reporting` module for more details.
 #[derive(Copy, Clone, Debug)]
 pub enum RegionVariableOrigin {
-    /// Region variables created for ill-categorized reasons,
-    /// mostly indicates places in need of refactoring
+    /// Region variables created for ill-categorized reasons.
+    ///
+    /// They mostly indicate places in need of refactoring.
     MiscVariable(Span),
 
-    /// Regions created by a `&P` or `[...]` pattern
+    /// Regions created by a `&P` or `[...]` pattern.
     PatternRegion(Span),
 
-    /// Regions created by `&` operator
+    /// Regions created by `&` operator.
+    ///
     AddrOfRegion(Span),
-
-    /// Regions created as part of an autoref of a method receiver
+    /// Regions created as part of an autoref of a method receiver.
     Autoref(Span),
 
-    /// Regions created as part of an automatic coercion
+    /// Regions created as part of an automatic coercion.
     Coercion(Span),
 
-    /// Region variables created as the values for early-bound regions
+    /// Region variables created as the values for early-bound regions.
     EarlyBoundRegion(Span, Symbol),
 
     /// Region variables created for bound regions
-    /// in a function or method that is called
+    /// in a function or method that is called.
     LateBoundRegion(Span, ty::BoundRegionKind, LateBoundRegionConversionTime),
 
     UpvarRegion(ty::UpvarId, Span),
@@ -534,7 +554,7 @@ impl<'tcx> fmt::Display for FixupError<'tcx> {
     }
 }
 
-/// Used to configure inference contexts before their creation
+/// Used to configure inference contexts before their creation.
 pub struct InferCtxtBuilder<'tcx> {
     tcx: TyCtxt<'tcx>,
     defining_use_anchor: DefiningAnchor,
@@ -570,8 +590,8 @@ impl<'tcx> InferCtxtBuilder<'tcx> {
         self
     }
 
-    pub fn intercrate(mut self) -> Self {
-        self.intercrate = true;
+    pub fn intercrate(mut self, intercrate: bool) -> Self {
+        self.intercrate = intercrate;
         self
     }
 
@@ -618,6 +638,7 @@ impl<'tcx> InferCtxtBuilder<'tcx> {
             skip_leak_check: Cell::new(false),
             universe: Cell::new(ty::UniverseIndex::ROOT),
             intercrate,
+            inside_canonicalization_ctxt: Cell::new(false),
         }
     }
 }
@@ -691,12 +712,7 @@ impl<'tcx> InferCtxt<'tcx> {
     }
 
     pub fn freshener<'b>(&'b self) -> TypeFreshener<'b, 'tcx> {
-        freshen::TypeFreshener::new(self, false)
-    }
-
-    /// Like `freshener`, but does not replace `'static` regions.
-    pub fn freshener_keep_static<'b>(&'b self) -> TypeFreshener<'b, 'tcx> {
-        freshen::TypeFreshener::new(self, true)
+        freshen::TypeFreshener::new(self)
     }
 
     pub fn unsolved_variables(&self) -> Vec<Ty<'tcx>> {
@@ -726,7 +742,7 @@ impl<'tcx> InferCtxt<'tcx> {
         &'a self,
         trace: TypeTrace<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        define_opaque_types: bool,
+        define_opaque_types: DefineOpaqueTypes,
     ) -> CombineFields<'a, 'tcx> {
         CombineFields {
             infcx: self,
@@ -835,9 +851,9 @@ impl<'tcx> InferCtxt<'tcx> {
 
     /// Scan the constraints produced since `snapshot` began and returns:
     ///
-    /// - `None` -- if none of them involve "region outlives" constraints
-    /// - `Some(true)` -- if there are `'a: 'b` constraints where `'a` or `'b` is a placeholder
-    /// - `Some(false)` -- if there are `'a: 'b` constraints but none involve placeholders
+    /// - `None` -- if none of them involves "region outlives" constraints.
+    /// - `Some(true)` -- if there are `'a: 'b` constraints where `'a` or `'b` is a placeholder.
+    /// - `Some(false)` -- if there are `'a: 'b` constraints but none involve placeholders.
     pub fn region_constraints_added_in_snapshot(
         &self,
         snapshot: &CombinedSnapshot<'tcx>,
@@ -852,16 +868,12 @@ impl<'tcx> InferCtxt<'tcx> {
         self.inner.borrow().undo_log.opaque_types_in_snapshot(&snapshot.undo_snapshot)
     }
 
-    pub fn add_given(&self, sub: ty::Region<'tcx>, sup: ty::RegionVid) {
-        self.inner.borrow_mut().unwrap_region_constraints().add_given(sub, sup);
-    }
-
     pub fn can_sub<T>(&self, param_env: ty::ParamEnv<'tcx>, a: T, b: T) -> bool
     where
         T: at::ToTrace<'tcx>,
     {
         let origin = &ObligationCause::dummy();
-        self.probe(|_| self.at(origin, param_env).sub(a, b).is_ok())
+        self.probe(|_| self.at(origin, param_env).sub(DefineOpaqueTypes::No, a, b).is_ok())
     }
 
     pub fn can_eq<T>(&self, param_env: ty::ParamEnv<'tcx>, a: T, b: T) -> bool
@@ -869,7 +881,7 @@ impl<'tcx> InferCtxt<'tcx> {
         T: at::ToTrace<'tcx>,
     {
         let origin = &ObligationCause::dummy();
-        self.probe(|_| self.at(origin, param_env).eq(a, b).is_ok())
+        self.probe(|_| self.at(origin, param_env).eq(DefineOpaqueTypes::No, a, b).is_ok())
     }
 
     #[instrument(skip(self), level = "debug")]
@@ -964,7 +976,8 @@ impl<'tcx> InferCtxt<'tcx> {
             let ty::SubtypePredicate { a_is_expected, a, b } =
                 self.instantiate_binder_with_placeholders(predicate);
 
-            let ok = self.at(cause, param_env).sub_exp(a_is_expected, a, b)?;
+            let ok =
+                self.at(cause, param_env).sub_exp(DefineOpaqueTypes::No, a_is_expected, a, b)?;
 
             Ok(ok.unit())
         }))
@@ -1199,95 +1212,6 @@ impl<'tcx> InferCtxt<'tcx> {
         self.tainted_by_errors.set(Some(e));
     }
 
-    pub fn skip_region_resolution(&self) {
-        let (var_infos, _) = {
-            let mut inner = self.inner.borrow_mut();
-            let inner = &mut *inner;
-            // Note: `inner.region_obligations` may not be empty, because we
-            // didn't necessarily call `process_registered_region_obligations`.
-            // This is okay, because that doesn't introduce new vars.
-            inner
-                .region_constraint_storage
-                .take()
-                .expect("regions already resolved")
-                .with_log(&mut inner.undo_log)
-                .into_infos_and_data()
-        };
-
-        let lexical_region_resolutions = LexicalRegionResolutions {
-            values: rustc_index::vec::IndexVec::from_elem_n(
-                crate::infer::lexical_region_resolve::VarValue::Value(self.tcx.lifetimes.re_erased),
-                var_infos.len(),
-            ),
-        };
-
-        let old_value = self.lexical_region_resolutions.replace(Some(lexical_region_resolutions));
-        assert!(old_value.is_none());
-    }
-
-    /// Process the region constraints and return any errors that
-    /// result. After this, no more unification operations should be
-    /// done -- or the compiler will panic -- but it is legal to use
-    /// `resolve_vars_if_possible` as well as `fully_resolve`.
-    pub fn resolve_regions(
-        &self,
-        outlives_env: &OutlivesEnvironment<'tcx>,
-    ) -> Vec<RegionResolutionError<'tcx>> {
-        let (var_infos, data) = {
-            let mut inner = self.inner.borrow_mut();
-            let inner = &mut *inner;
-            assert!(
-                self.tainted_by_errors().is_some() || inner.region_obligations.is_empty(),
-                "region_obligations not empty: {:#?}",
-                inner.region_obligations
-            );
-            inner
-                .region_constraint_storage
-                .take()
-                .expect("regions already resolved")
-                .with_log(&mut inner.undo_log)
-                .into_infos_and_data()
-        };
-
-        let region_rels = &RegionRelations::new(self.tcx, outlives_env.free_region_map());
-
-        let (lexical_region_resolutions, errors) =
-            lexical_region_resolve::resolve(outlives_env.param_env, region_rels, var_infos, data);
-
-        let old_value = self.lexical_region_resolutions.replace(Some(lexical_region_resolutions));
-        assert!(old_value.is_none());
-
-        errors
-    }
-    /// Obtains (and clears) the current set of region
-    /// constraints. The inference context is still usable: further
-    /// unifications will simply add new constraints.
-    ///
-    /// This method is not meant to be used with normal lexical region
-    /// resolution. Rather, it is used in the NLL mode as a kind of
-    /// interim hack: basically we run normal type-check and generate
-    /// region constraints as normal, but then we take them and
-    /// translate them into the form that the NLL solver
-    /// understands. See the NLL module for mode details.
-    pub fn take_and_reset_region_constraints(&self) -> RegionConstraintData<'tcx> {
-        assert!(
-            self.inner.borrow().region_obligations.is_empty(),
-            "region_obligations not empty: {:#?}",
-            self.inner.borrow().region_obligations
-        );
-
-        self.inner.borrow_mut().unwrap_region_constraints().take_and_reset_data()
-    }
-
-    /// Gives temporary access to the region constraint data.
-    pub fn with_region_constraints<R>(
-        &self,
-        op: impl FnOnce(&RegionConstraintData<'tcx>) -> R,
-    ) -> R {
-        let mut inner = self.inner.borrow_mut();
-        op(inner.unwrap_region_constraints().data())
-    }
-
     pub fn region_var_origin(&self, vid: ty::RegionVid) -> RegionVariableOrigin {
         let mut inner = self.inner.borrow_mut();
         let inner = &mut *inner;
@@ -1356,6 +1280,32 @@ impl<'tcx> InferCtxt<'tcx> {
         self.inner.borrow_mut().type_variables().root_var(var)
     }
 
+    pub fn root_const_var(&self, var: ty::ConstVid<'tcx>) -> ty::ConstVid<'tcx> {
+        self.inner.borrow_mut().const_unification_table().find(var)
+    }
+
+    /// Resolves an int var to a rigid int type, if it was constrained to one,
+    /// or else the root int var in the unification table.
+    pub fn opportunistic_resolve_int_var(&self, vid: ty::IntVid) -> Ty<'tcx> {
+        let mut inner = self.inner.borrow_mut();
+        if let Some(value) = inner.int_unification_table().probe_value(vid) {
+            value.to_type(self.tcx)
+        } else {
+            self.tcx.mk_int_var(inner.int_unification_table().find(vid))
+        }
+    }
+
+    /// Resolves a float var to a rigid int type, if it was constrained to one,
+    /// or else the root float var in the unification table.
+    pub fn opportunistic_resolve_float_var(&self, vid: ty::FloatVid) -> Ty<'tcx> {
+        let mut inner = self.inner.borrow_mut();
+        if let Some(value) = inner.float_unification_table().probe_value(vid) {
+            value.to_type(self.tcx)
+        } else {
+            self.tcx.mk_float_var(inner.float_unification_table().find(vid))
+        }
+    }
+
     /// Where possible, replaces type/const variables in
     /// `value` with their final value. Note that region variables
     /// are unaffected. If a type/const variable has not been unified, it
@@ -1614,6 +1564,28 @@ impl<'tcx> InferCtxt<'tcx> {
         tcx.const_eval_resolve_for_typeck(param_env_erased, unevaluated, span)
     }
 
+    /// The returned function is used in a fast path. If it returns `true` the variable is
+    /// unchanged, `false` indicates that the status is unknown.
+    #[inline]
+    pub fn is_ty_infer_var_definitely_unchanged<'a>(
+        &'a self,
+    ) -> (impl Fn(TyOrConstInferVar<'tcx>) -> bool + 'a) {
+        // This hoists the borrow/release out of the loop body.
+        let inner = self.inner.try_borrow();
+
+        return move |infer_var: TyOrConstInferVar<'tcx>| match (infer_var, &inner) {
+            (TyOrConstInferVar::Ty(ty_var), Ok(inner)) => {
+                use self::type_variable::TypeVariableValue;
+
+                match inner.try_type_variables_probe_ref(ty_var) {
+                    Some(TypeVariableValue::Unknown { .. }) => true,
+                    _ => false,
+                }
+            }
+            _ => false,
+        };
+    }
+
     /// `ty_or_const_infer_var_changed` is equivalent to one of these two:
     ///   * `shallow_resolve(ty) != ty` (where `ty.kind = ty::Infer(_)`)
     ///   * `shallow_resolve(ct) != ct` (where `ct.kind = ty::ConstKind::Infer(_)`)
@@ -1664,59 +1636,34 @@ impl<'tcx> InferCtxt<'tcx> {
             }
         }
     }
-}
 
-impl<'tcx> TypeErrCtxt<'_, 'tcx> {
-    /// Processes registered region obliations and resolves regions, reporting
-    /// any errors if any were raised. Prefer using this function over manually
-    /// calling `resolve_regions_and_report_errors`.
-    pub fn check_region_obligations_and_report_errors(
-        &self,
-        generic_param_scope: LocalDefId,
-        outlives_env: &OutlivesEnvironment<'tcx>,
-    ) -> Result<(), ErrorGuaranteed> {
-        self.process_registered_region_obligations(
-            outlives_env.region_bound_pairs(),
-            outlives_env.param_env,
-        );
+    pub fn inside_canonicalization_ctxt(&self) -> bool {
+        self.inside_canonicalization_ctxt.get()
+    }
 
-        self.resolve_regions_and_report_errors(generic_param_scope, outlives_env)
+    pub fn set_canonicalization_ctxt(&self) -> CanonicalizationCtxtGuard<'_, 'tcx> {
+        let prev_ctxt = self.inside_canonicalization_ctxt();
+        self.inside_canonicalization_ctxt.set(true);
+        CanonicalizationCtxtGuard { prev_ctxt, infcx: self }
     }
 
-    /// Process the region constraints and report any errors that
-    /// result. After this, no more unification operations should be
-    /// done -- or the compiler will panic -- but it is legal to use
-    /// `resolve_vars_if_possible` as well as `fully_resolve`.
-    ///
-    /// Make sure to call [`InferCtxt::process_registered_region_obligations`]
-    /// first, or preferably use [`TypeErrCtxt::check_region_obligations_and_report_errors`]
-    /// to do both of these operations together.
-    pub fn resolve_regions_and_report_errors(
-        &self,
-        generic_param_scope: LocalDefId,
-        outlives_env: &OutlivesEnvironment<'tcx>,
-    ) -> Result<(), ErrorGuaranteed> {
-        let errors = self.resolve_regions(outlives_env);
-
-        if let None = self.tainted_by_errors() {
-            // As a heuristic, just skip reporting region errors
-            // altogether if other errors have been reported while
-            // this infcx was in use. This is totally hokey but
-            // otherwise we have a hard time separating legit region
-            // errors from silly ones.
-            self.report_region_errors(generic_param_scope, &errors);
-        }
+    fn set_canonicalization_ctxt_to(&self, ctxt: bool) {
+        self.inside_canonicalization_ctxt.set(ctxt);
+    }
+}
 
-        if errors.is_empty() {
-            Ok(())
-        } else {
-            Err(self
-                .tcx
-                .sess
-                .delay_span_bug(rustc_span::DUMMY_SP, "error should have been emitted"))
-        }
+pub struct CanonicalizationCtxtGuard<'cx, 'tcx> {
+    prev_ctxt: bool,
+    infcx: &'cx InferCtxt<'tcx>,
+}
+
+impl<'cx, 'tcx> Drop for CanonicalizationCtxtGuard<'cx, 'tcx> {
+    fn drop(&mut self) {
+        self.infcx.set_canonicalization_ctxt_to(self.prev_ctxt)
     }
+}
 
+impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     // [Note-Type-error-reporting]
     // An invariant is that anytime the expected or actual type is Error (the special
     // error type, meaning that an error occurred when typechecking this expression),
@@ -1770,7 +1717,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     }
 }
 
-/// Helper for `ty_or_const_infer_var_changed` (see comment on that), currently
+/// Helper for [InferCtxt::ty_or_const_infer_var_changed] (see comment on that), currently
 /// used only for `traits::fulfill`'s list of `stalled_on` inference variables.
 #[derive(Copy, Clone, Debug)]
 pub enum TyOrConstInferVar<'tcx> {
@@ -2043,13 +1990,17 @@ fn replace_param_and_infer_substs_with_placeholder<'tcx>(
 
         fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
             if let ty::Infer(_) = t.kind() {
+                let idx = {
+                    let idx = self.idx;
+                    self.idx += 1;
+                    idx
+                };
                 self.tcx.mk_placeholder(ty::PlaceholderType {
                     universe: ty::UniverseIndex::ROOT,
-                    name: ty::BoundTyKind::Anon({
-                        let idx = self.idx;
-                        self.idx += 1;
-                        idx
-                    }),
+                    bound: ty::BoundTy {
+                        var: ty::BoundVar::from_u32(idx),
+                        kind: ty::BoundTyKind::Anon,
+                    },
                 })
             } else {
                 t.super_fold_with(self)
@@ -2066,7 +2017,7 @@ fn replace_param_and_infer_substs_with_placeholder<'tcx>(
                 self.tcx.mk_const(
                     ty::PlaceholderConst {
                         universe: ty::UniverseIndex::ROOT,
-                        name: ty::BoundVar::from_u32({
+                        bound: ty::BoundVar::from_u32({
                             let idx = self.idx;
                             self.idx += 1;
                             idx
diff --git a/compiler/rustc_infer/src/infer/nll_relate/mod.rs b/compiler/rustc_infer/src/infer/nll_relate/mod.rs
index 573cd91a2..9f7b26b87 100644
--- a/compiler/rustc_infer/src/infer/nll_relate/mod.rs
+++ b/compiler/rustc_infer/src/infer/nll_relate/mod.rs
@@ -28,6 +28,7 @@ use crate::traits::{Obligation, PredicateObligations};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::error::TypeError;
+use rustc_middle::ty::fold::FnMutDelegate;
 use rustc_middle::ty::relate::{self, Relate, RelateResult, TypeRelation};
 use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor};
 use rustc_middle::ty::{self, InferConst, Ty, TyCtxt};
@@ -55,21 +56,6 @@ where
     ambient_variance: ty::Variance,
 
     ambient_variance_info: ty::VarianceDiagInfo<'tcx>,
-
-    /// When we pass through a set of binders (e.g., when looking into
-    /// a `fn` type), we push a new bound region scope onto here. This
-    /// will contain the instantiated region for each region in those
-    /// binders. When we then encounter a `ReLateBound(d, br)`, we can
-    /// use the De Bruijn index `d` to find the right scope, and then
-    /// bound region name `br` to find the specific instantiation from
-    /// within that scope. See `replace_bound_region`.
-    ///
-    /// This field stores the instantiations for late-bound regions in
-    /// the `a` type.
-    a_scopes: Vec<BoundRegionScope<'tcx>>,
-
-    /// Same as `a_scopes`, but for the `b` type.
-    b_scopes: Vec<BoundRegionScope<'tcx>>,
 }
 
 pub trait TypeRelatingDelegate<'tcx> {
@@ -147,8 +133,6 @@ where
             delegate,
             ambient_variance,
             ambient_variance_info: ty::VarianceDiagInfo::default(),
-            a_scopes: vec![],
-            b_scopes: vec![],
         }
     }
 
@@ -166,88 +150,6 @@ where
         }
     }
 
-    fn create_scope(
-        &mut self,
-        value: ty::Binder<'tcx, impl Relate<'tcx>>,
-        universally_quantified: UniversallyQuantified,
-    ) -> BoundRegionScope<'tcx> {
-        let mut scope = BoundRegionScope::default();
-
-        // Create a callback that creates (via the delegate) either an
-        // existential or placeholder region as needed.
-        let mut next_region = {
-            let delegate = &mut self.delegate;
-            let mut lazy_universe = None;
-            move |br: ty::BoundRegion| {
-                if universally_quantified.0 {
-                    // The first time this closure is called, create a
-                    // new universe for the placeholders we will make
-                    // from here out.
-                    let universe = lazy_universe.unwrap_or_else(|| {
-                        let universe = delegate.create_next_universe();
-                        lazy_universe = Some(universe);
-                        universe
-                    });
-
-                    let placeholder = ty::PlaceholderRegion { universe, name: br.kind };
-                    delegate.next_placeholder_region(placeholder)
-                } else {
-                    delegate.next_existential_region_var(true, br.kind.get_name())
-                }
-            }
-        };
-
-        value.skip_binder().visit_with(&mut ScopeInstantiator {
-            next_region: &mut next_region,
-            target_index: ty::INNERMOST,
-            bound_region_scope: &mut scope,
-        });
-
-        scope
-    }
-
-    /// When we encounter binders during the type traversal, we record
-    /// the value to substitute for each of the things contained in
-    /// that binder. (This will be either a universal placeholder or
-    /// an existential inference variable.) Given the De Bruijn index
-    /// `debruijn` (and name `br`) of some binder we have now
-    /// encountered, this routine finds the value that we instantiated
-    /// the region with; to do so, it indexes backwards into the list
-    /// of ambient scopes `scopes`.
-    fn lookup_bound_region(
-        debruijn: ty::DebruijnIndex,
-        br: &ty::BoundRegion,
-        first_free_index: ty::DebruijnIndex,
-        scopes: &[BoundRegionScope<'tcx>],
-    ) -> ty::Region<'tcx> {
-        // The debruijn index is a "reverse index" into the
-        // scopes listing. So when we have INNERMOST (0), we
-        // want the *last* scope pushed, and so forth.
-        let debruijn_index = debruijn.index() - first_free_index.index();
-        let scope = &scopes[scopes.len() - debruijn_index - 1];
-
-        // Find this bound region in that scope to map to a
-        // particular region.
-        scope.map[br]
-    }
-
-    /// If `r` is a bound region, find the scope in which it is bound
-    /// (from `scopes`) and return the value that we instantiated it
-    /// with. Otherwise just return `r`.
-    fn replace_bound_region(
-        &self,
-        r: ty::Region<'tcx>,
-        first_free_index: ty::DebruijnIndex,
-        scopes: &[BoundRegionScope<'tcx>],
-    ) -> ty::Region<'tcx> {
-        debug!("replace_bound_regions(scopes={:?})", scopes);
-        if let ty::ReLateBound(debruijn, br) = *r {
-            Self::lookup_bound_region(debruijn, &br, first_free_index, scopes)
-        } else {
-            r
-        }
-    }
-
     /// Push a new outlives requirement into our output set of
     /// constraints.
     fn push_outlives(
@@ -314,18 +216,9 @@ where
 
         self.infcx.inner.borrow_mut().type_variables().instantiate(vid, generalized_ty);
 
-        // The generalized values we extract from `canonical_var_values` have
-        // been fully instantiated and hence the set of scopes we have
-        // doesn't matter -- just to be sure, put an empty vector
-        // in there.
-        let old_a_scopes = std::mem::take(pair.vid_scopes(self));
-
         // Relate the generalized kind to the original one.
         let result = pair.relate_generalized_ty(self, generalized_ty);
 
-        // Restore the old scopes now.
-        *pair.vid_scopes(self) = old_a_scopes;
-
         debug!("relate_ty_var: complete, result = {:?}", result);
         result
     }
@@ -379,6 +272,97 @@ where
         trace!(a = ?a.kind(), b = ?b.kind(), "opaque type instantiated");
         Ok(a)
     }
+
+    #[instrument(skip(self), level = "debug")]
+    fn instantiate_binder_with_placeholders<T>(&mut self, binder: ty::Binder<'tcx, T>) -> T
+    where
+        T: ty::TypeFoldable<TyCtxt<'tcx>> + Copy,
+    {
+        if let Some(inner) = binder.no_bound_vars() {
+            return inner;
+        }
+
+        let mut next_region = {
+            let nll_delegate = &mut self.delegate;
+            let mut lazy_universe = None;
+
+            move |br: ty::BoundRegion| {
+                // The first time this closure is called, create a
+                // new universe for the placeholders we will make
+                // from here out.
+                let universe = lazy_universe.unwrap_or_else(|| {
+                    let universe = nll_delegate.create_next_universe();
+                    lazy_universe = Some(universe);
+                    universe
+                });
+
+                let placeholder = ty::PlaceholderRegion { universe, bound: br };
+                debug!(?placeholder);
+                let placeholder_reg = nll_delegate.next_placeholder_region(placeholder);
+                debug!(?placeholder_reg);
+
+                placeholder_reg
+            }
+        };
+
+        let delegate = FnMutDelegate {
+            regions: &mut next_region,
+            types: &mut |_bound_ty: ty::BoundTy| {
+                unreachable!("we only replace regions in nll_relate, not types")
+            },
+            consts: &mut |_bound_var: ty::BoundVar, _ty| {
+                unreachable!("we only replace regions in nll_relate, not consts")
+            },
+        };
+
+        let replaced = self.infcx.tcx.replace_bound_vars_uncached(binder, delegate);
+        debug!(?replaced);
+
+        replaced
+    }
+
+    #[instrument(skip(self), level = "debug")]
+    fn instantiate_binder_with_existentials<T>(&mut self, binder: ty::Binder<'tcx, T>) -> T
+    where
+        T: ty::TypeFoldable<TyCtxt<'tcx>> + Copy,
+    {
+        if let Some(inner) = binder.no_bound_vars() {
+            return inner;
+        }
+
+        let mut next_region = {
+            let nll_delegate = &mut self.delegate;
+            let mut reg_map = FxHashMap::default();
+
+            move |br: ty::BoundRegion| {
+                if let Some(ex_reg_var) = reg_map.get(&br) {
+                    return *ex_reg_var;
+                } else {
+                    let ex_reg_var =
+                        nll_delegate.next_existential_region_var(true, br.kind.get_name());
+                    debug!(?ex_reg_var);
+                    reg_map.insert(br, ex_reg_var);
+
+                    ex_reg_var
+                }
+            }
+        };
+
+        let delegate = FnMutDelegate {
+            regions: &mut next_region,
+            types: &mut |_bound_ty: ty::BoundTy| {
+                unreachable!("we only replace regions in nll_relate, not types")
+            },
+            consts: &mut |_bound_var: ty::BoundVar, _ty| {
+                unreachable!("we only replace regions in nll_relate, not consts")
+            },
+        };
+
+        let replaced = self.infcx.tcx.replace_bound_vars_uncached(binder, delegate);
+        debug!(?replaced);
+
+        replaced
+    }
 }
 
 /// When we instantiate an inference variable with a value in
@@ -396,14 +380,6 @@ trait VidValuePair<'tcx>: Debug {
     /// opposite part of the tuple from the vid).
     fn value_ty(&self) -> Ty<'tcx>;
 
-    /// Extract the scopes that apply to whichever side of the tuple
-    /// the vid was found on. See the comment where this is called
-    /// for more details on why we want them.
-    fn vid_scopes<'r, D: TypeRelatingDelegate<'tcx>>(
-        &self,
-        relate: &'r mut TypeRelating<'_, 'tcx, D>,
-    ) -> &'r mut Vec<BoundRegionScope<'tcx>>;
-
     /// Given a generalized type G that should replace the vid, relate
     /// G to the value, putting G on whichever side the vid would have
     /// appeared.
@@ -425,16 +401,6 @@ impl<'tcx> VidValuePair<'tcx> for (ty::TyVid, Ty<'tcx>) {
         self.1
     }
 
-    fn vid_scopes<'r, D>(
-        &self,
-        relate: &'r mut TypeRelating<'_, 'tcx, D>,
-    ) -> &'r mut Vec<BoundRegionScope<'tcx>>
-    where
-        D: TypeRelatingDelegate<'tcx>,
-    {
-        &mut relate.a_scopes
-    }
-
     fn relate_generalized_ty<D>(
         &self,
         relate: &mut TypeRelating<'_, 'tcx, D>,
@@ -457,16 +423,6 @@ impl<'tcx> VidValuePair<'tcx> for (Ty<'tcx>, ty::TyVid) {
         self.0
     }
 
-    fn vid_scopes<'r, D>(
-        &self,
-        relate: &'r mut TypeRelating<'_, 'tcx, D>,
-    ) -> &'r mut Vec<BoundRegionScope<'tcx>>
-    where
-        D: TypeRelatingDelegate<'tcx>,
-    {
-        &mut relate.b_scopes
-    }
-
     fn relate_generalized_ty<D>(
         &self,
         relate: &mut TypeRelating<'_, 'tcx, D>,
@@ -487,10 +443,6 @@ where
         self.infcx.tcx
     }
 
-    fn intercrate(&self) -> bool {
-        self.infcx.intercrate
-    }
-
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
         self.delegate.param_env()
     }
@@ -503,17 +455,6 @@ where
         true
     }
 
-    fn mark_ambiguous(&mut self) {
-        let cause = ObligationCause::dummy_with_span(self.delegate.span());
-        let param_env = self.delegate.param_env();
-        self.delegate.register_obligations(vec![Obligation::new(
-            self.tcx(),
-            cause,
-            param_env,
-            ty::Binder::dummy(ty::PredicateKind::Ambiguous),
-        )]);
-    }
-
     #[instrument(skip(self, info), level = "trace", ret)]
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
@@ -602,20 +543,14 @@ where
     ) -> RelateResult<'tcx, ty::Region<'tcx>> {
         debug!(?self.ambient_variance);
 
-        let v_a = self.replace_bound_region(a, ty::INNERMOST, &self.a_scopes);
-        let v_b = self.replace_bound_region(b, ty::INNERMOST, &self.b_scopes);
-
-        debug!(?v_a);
-        debug!(?v_b);
-
         if self.ambient_covariance() {
             // Covariant: &'a u8 <: &'b u8. Hence, `'a: 'b`.
-            self.push_outlives(v_a, v_b, self.ambient_variance_info);
+            self.push_outlives(a, b, self.ambient_variance_info);
         }
 
         if self.ambient_contravariance() {
             // Contravariant: &'b u8 <: &'a u8. Hence, `'b: 'a`.
-            self.push_outlives(v_b, v_a, self.ambient_variance_info);
+            self.push_outlives(b, a, self.ambient_variance_info);
         }
 
         Ok(a)
@@ -689,15 +624,6 @@ where
             // instantiation of B (i.e., B instantiated with
             // universals).
 
-            let b_scope = self.create_scope(b, UniversallyQuantified(true));
-            let a_scope = self.create_scope(a, UniversallyQuantified(false));
-
-            debug!(?a_scope, "(existential)");
-            debug!(?b_scope, "(universal)");
-
-            self.b_scopes.push(b_scope);
-            self.a_scopes.push(a_scope);
-
             // Reset the ambient variance to covariant. This is needed
             // to correctly handle cases like
             //
@@ -718,12 +644,14 @@ where
             //   subtyping (i.e., `&'b u32 <: &{P} u32`).
             let variance = std::mem::replace(&mut self.ambient_variance, ty::Variance::Covariant);
 
-            self.relate(a.skip_binder(), b.skip_binder())?;
+            // Note: the order here is important. Create the placeholders first, otherwise
+            // we assign the wrong universe to the existential!
+            let b_replaced = self.instantiate_binder_with_placeholders(b);
+            let a_replaced = self.instantiate_binder_with_existentials(a);
 
-            self.ambient_variance = variance;
+            self.relate(a_replaced, b_replaced)?;
 
-            self.b_scopes.pop().unwrap();
-            self.a_scopes.pop().unwrap();
+            self.ambient_variance = variance;
         }
 
         if self.ambient_contravariance() {
@@ -733,26 +661,17 @@ where
             // instantiation of B (i.e., B instantiated with
             // existentials). Opposite of above.
 
-            let a_scope = self.create_scope(a, UniversallyQuantified(true));
-            let b_scope = self.create_scope(b, UniversallyQuantified(false));
-
-            debug!(?a_scope, "(universal)");
-            debug!(?b_scope, "(existential)");
-
-            self.a_scopes.push(a_scope);
-            self.b_scopes.push(b_scope);
-
             // Reset ambient variance to contravariance. See the
             // covariant case above for an explanation.
             let variance =
                 std::mem::replace(&mut self.ambient_variance, ty::Variance::Contravariant);
 
-            self.relate(a.skip_binder(), b.skip_binder())?;
+            let a_replaced = self.instantiate_binder_with_placeholders(a);
+            let b_replaced = self.instantiate_binder_with_existentials(b);
 
-            self.ambient_variance = variance;
+            self.relate(a_replaced, b_replaced)?;
 
-            self.b_scopes.pop().unwrap();
-            self.a_scopes.pop().unwrap();
+            self.ambient_variance = variance;
         }
 
         Ok(a)
@@ -777,6 +696,34 @@ where
     fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
         self.delegate.register_obligations(obligations);
     }
+
+    fn alias_relate_direction(&self) -> ty::AliasRelationDirection {
+        unreachable!("manually overridden to handle ty::Variance::Contravariant ambient variance")
+    }
+
+    fn register_type_relate_obligation(&mut self, a: Ty<'tcx>, b: Ty<'tcx>) {
+        self.register_predicates([ty::Binder::dummy(match self.ambient_variance {
+            ty::Variance::Covariant => ty::PredicateKind::AliasRelate(
+                a.into(),
+                b.into(),
+                ty::AliasRelationDirection::Subtype,
+            ),
+            // a :> b is b <: a
+            ty::Variance::Contravariant => ty::PredicateKind::AliasRelate(
+                b.into(),
+                a.into(),
+                ty::AliasRelationDirection::Subtype,
+            ),
+            ty::Variance::Invariant => ty::PredicateKind::AliasRelate(
+                a.into(),
+                b.into(),
+                ty::AliasRelationDirection::Equate,
+            ),
+            // FIXME(deferred_projection_equality): Implement this when we trigger it.
+            // Probably just need to do nothing here.
+            ty::Variance::Bivariant => unreachable!(),
+        })]);
+    }
 }
 
 /// When we encounter a binder like `for<..> fn(..)`, we actually have
@@ -872,11 +819,6 @@ where
         self.infcx.tcx
     }
 
-    fn intercrate(&self) -> bool {
-        assert!(!self.infcx.intercrate);
-        false
-    }
-
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
         self.delegate.param_env()
     }
@@ -889,10 +831,6 @@ where
         true
     }
 
-    fn mark_ambiguous(&mut self) {
-        bug!()
-    }
-
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
         variance: ty::Variance,
diff --git a/compiler/rustc_infer/src/infer/opaque_types.rs b/compiler/rustc_infer/src/infer/opaque_types.rs
index d5c824d4c..3a0a0494a 100644
--- a/compiler/rustc_infer/src/infer/opaque_types.rs
+++ b/compiler/rustc_infer/src/infer/opaque_types.rs
@@ -1,11 +1,12 @@
+use super::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
+use super::{DefineOpaqueTypes, InferResult};
 use crate::errors::OpaqueHiddenTypeDiag;
 use crate::infer::{DefiningAnchor, InferCtxt, InferOk};
 use crate::traits;
-use hir::def::DefKind;
 use hir::def_id::{DefId, LocalDefId};
 use hir::OpaqueTyOrigin;
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::Lrc;
-use rustc_data_structures::vec_map::VecMap;
 use rustc_hir as hir;
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
@@ -16,18 +17,13 @@ use rustc_middle::ty::{
     TypeVisitable, TypeVisitableExt, TypeVisitor,
 };
 use rustc_span::Span;
-
 use std::ops::ControlFlow;
 
-pub type OpaqueTypeMap<'tcx> = VecMap<OpaqueTypeKey<'tcx>, OpaqueTypeDecl<'tcx>>;
-
 mod table;
 
+pub type OpaqueTypeMap<'tcx> = FxIndexMap<OpaqueTypeKey<'tcx>, OpaqueTypeDecl<'tcx>>;
 pub use table::{OpaqueTypeStorage, OpaqueTypeTable};
 
-use super::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use super::InferResult;
-
 /// Information about the opaque types whose values we
 /// are inferring in this function (these are the `impl Trait` that
 /// appear in the return type).
@@ -481,9 +477,7 @@ where
                 }
             }
 
-            ty::Alias(ty::Projection, proj)
-                if self.tcx.def_kind(proj.def_id) == DefKind::ImplTraitPlaceholder =>
-            {
+            ty::Alias(ty::Projection, proj) if self.tcx.is_impl_trait_in_trait(proj.def_id) => {
                 // Skip lifetime parameters that are not captures.
                 let variances = self.tcx.variances_of(proj.def_id);
 
@@ -547,8 +541,7 @@ impl<'tcx> InferCtxt<'tcx> {
         if let Some(prev) = prev {
             obligations = self
                 .at(&cause, param_env)
-                .define_opaque_types(true)
-                .eq_exp(a_is_expected, prev, hidden_ty)?
+                .eq_exp(DefineOpaqueTypes::Yes, a_is_expected, prev, hidden_ty)?
                 .obligations;
         }
 
@@ -563,8 +556,7 @@ impl<'tcx> InferCtxt<'tcx> {
                     // FIXME(RPITIT): Don't replace RPITITs with inference vars.
                     ty::Alias(ty::Projection, projection_ty)
                         if !projection_ty.has_escaping_bound_vars()
-                            && tcx.def_kind(projection_ty.def_id)
-                                != DefKind::ImplTraitPlaceholder =>
+                            && !tcx.is_impl_trait_in_trait(projection_ty.def_id) =>
                     {
                         self.infer_projection(
                             param_env,
diff --git a/compiler/rustc_infer/src/infer/outlives/env.rs b/compiler/rustc_infer/src/infer/outlives/env.rs
index 24e3c34dd..47e3dd762 100644
--- a/compiler/rustc_infer/src/infer/outlives/env.rs
+++ b/compiler/rustc_infer/src/infer/outlives/env.rs
@@ -1,9 +1,9 @@
 use crate::infer::free_regions::FreeRegionMap;
-use crate::infer::{GenericKind, InferCtxt};
+use crate::infer::GenericKind;
 use crate::traits::query::OutlivesBound;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::transitive_relation::TransitiveRelationBuilder;
-use rustc_middle::ty::{self, ReEarlyBound, ReFree, ReVar, Region};
+use rustc_middle::ty::{self, Region};
 
 use super::explicit_outlives_bounds;
 
@@ -75,7 +75,7 @@ impl<'tcx> OutlivesEnvironment<'tcx> {
             region_bound_pairs: Default::default(),
         };
 
-        builder.add_outlives_bounds(None, explicit_outlives_bounds(param_env));
+        builder.add_outlives_bounds(explicit_outlives_bounds(param_env));
 
         builder
     }
@@ -89,11 +89,10 @@ impl<'tcx> OutlivesEnvironment<'tcx> {
     /// Create a new `OutlivesEnvironment` with extra outlives bounds.
     pub fn with_bounds(
         param_env: ty::ParamEnv<'tcx>,
-        infcx: Option<&InferCtxt<'tcx>>,
         extra_bounds: impl IntoIterator<Item = OutlivesBound<'tcx>>,
     ) -> Self {
         let mut builder = Self::builder(param_env);
-        builder.add_outlives_bounds(infcx, extra_bounds);
+        builder.add_outlives_bounds(extra_bounds);
         builder.build()
     }
 
@@ -120,12 +119,7 @@ impl<'tcx> OutlivesEnvironmentBuilder<'tcx> {
     }
 
     /// Processes outlives bounds that are known to hold, whether from implied or other sources.
-    ///
-    /// The `infcx` parameter is optional; if the implied bounds may
-    /// contain inference variables, it must be supplied, in which
-    /// case we will register "givens" on the inference context. (See
-    /// `RegionConstraintData`.)
-    fn add_outlives_bounds<I>(&mut self, infcx: Option<&InferCtxt<'tcx>>, outlives_bounds: I)
+    fn add_outlives_bounds<I>(&mut self, outlives_bounds: I)
     where
         I: IntoIterator<Item = OutlivesBound<'tcx>>,
     {
@@ -142,27 +136,17 @@ impl<'tcx> OutlivesEnvironmentBuilder<'tcx> {
                     self.region_bound_pairs
                         .insert(ty::OutlivesPredicate(GenericKind::Alias(alias_b), r_a));
                 }
-                OutlivesBound::RegionSubRegion(r_a, r_b) => {
-                    if let (ReEarlyBound(_) | ReFree(_), ReVar(vid_b)) = (r_a.kind(), r_b.kind()) {
-                        infcx
-                            .expect("no infcx provided but region vars found")
-                            .add_given(r_a, vid_b);
-                    } else {
-                        // In principle, we could record (and take
-                        // advantage of) every relationship here, but
-                        // we are also free not to -- it simply means
-                        // strictly less that we can successfully type
-                        // check. Right now we only look for things
-                        // relationships between free regions. (It may
-                        // also be that we should revise our inference
-                        // system to be more general and to make use
-                        // of *every* relationship that arises here,
-                        // but presently we do not.)
-                        if r_a.is_free_or_static() && r_b.is_free() {
-                            self.region_relation.add(r_a, r_b)
-                        }
-                    }
-                }
+                OutlivesBound::RegionSubRegion(r_a, r_b) => match (*r_a, *r_b) {
+                    (
+                        ty::ReStatic | ty::ReEarlyBound(_) | ty::ReFree(_),
+                        ty::ReStatic | ty::ReEarlyBound(_) | ty::ReFree(_),
+                    ) => self.region_relation.add(r_a, r_b),
+                    (ty::ReError(_), _) | (_, ty::ReError(_)) => {}
+                    // FIXME(#109628): We shouldn't have existential variables in implied bounds.
+                    // Panic here once the linked issue is resolved!
+                    (ty::ReVar(_), _) | (_, ty::ReVar(_)) => {}
+                    _ => bug!("add_outlives_bounds: unexpected regions: ({r_a:?}, {r_b:?})"),
+                },
             }
         }
     }
diff --git a/compiler/rustc_infer/src/infer/outlives/mod.rs b/compiler/rustc_infer/src/infer/outlives/mod.rs
index 83f3d5a74..9a9a1696b 100644
--- a/compiler/rustc_infer/src/infer/outlives/mod.rs
+++ b/compiler/rustc_infer/src/infer/outlives/mod.rs
@@ -1,4 +1,11 @@
 //! Various code related to computing outlives relations.
+use self::env::OutlivesEnvironment;
+use super::region_constraints::RegionConstraintData;
+use super::{InferCtxt, RegionResolutionError};
+use crate::infer::free_regions::RegionRelations;
+use crate::infer::lexical_region_resolve::{self, LexicalRegionResolutions};
+use rustc_middle::traits::query::OutlivesBound;
+use rustc_middle::ty;
 
 pub mod components;
 pub mod env;
@@ -6,9 +13,6 @@ pub mod obligations;
 pub mod test_type_match;
 pub mod verify;
 
-use rustc_middle::traits::query::OutlivesBound;
-use rustc_middle::ty;
-
 #[instrument(level = "debug", skip(param_env), ret)]
 pub fn explicit_outlives_bounds<'tcx>(
     param_env: ty::ParamEnv<'tcx>,
@@ -22,7 +26,7 @@ pub fn explicit_outlives_bounds<'tcx>(
             ty::PredicateKind::Clause(ty::Clause::Projection(..))
             | ty::PredicateKind::Clause(ty::Clause::Trait(..))
             | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
-            | ty::PredicateKind::AliasEq(..)
+            | ty::PredicateKind::AliasRelate(..)
             | ty::PredicateKind::Coerce(..)
             | ty::PredicateKind::Subtype(..)
             | ty::PredicateKind::WellFormed(..)
@@ -39,3 +43,98 @@ pub fn explicit_outlives_bounds<'tcx>(
             ))) => Some(OutlivesBound::RegionSubRegion(r_b, r_a)),
         })
 }
+
+impl<'tcx> InferCtxt<'tcx> {
+    pub fn skip_region_resolution(&self) {
+        let (var_infos, _) = {
+            let mut inner = self.inner.borrow_mut();
+            let inner = &mut *inner;
+            // Note: `inner.region_obligations` may not be empty, because we
+            // didn't necessarily call `process_registered_region_obligations`.
+            // This is okay, because that doesn't introduce new vars.
+            inner
+                .region_constraint_storage
+                .take()
+                .expect("regions already resolved")
+                .with_log(&mut inner.undo_log)
+                .into_infos_and_data()
+        };
+
+        let lexical_region_resolutions = LexicalRegionResolutions {
+            values: rustc_index::vec::IndexVec::from_elem_n(
+                crate::infer::lexical_region_resolve::VarValue::Value(self.tcx.lifetimes.re_erased),
+                var_infos.len(),
+            ),
+        };
+
+        let old_value = self.lexical_region_resolutions.replace(Some(lexical_region_resolutions));
+        assert!(old_value.is_none());
+    }
+
+    /// Process the region constraints and return any errors that
+    /// result. After this, no more unification operations should be
+    /// done -- or the compiler will panic -- but it is legal to use
+    /// `resolve_vars_if_possible` as well as `fully_resolve`.
+    #[must_use]
+    pub fn resolve_regions(
+        &self,
+        outlives_env: &OutlivesEnvironment<'tcx>,
+    ) -> Vec<RegionResolutionError<'tcx>> {
+        self.process_registered_region_obligations(outlives_env);
+
+        let (var_infos, data) = {
+            let mut inner = self.inner.borrow_mut();
+            let inner = &mut *inner;
+            assert!(
+                self.tainted_by_errors().is_some() || inner.region_obligations.is_empty(),
+                "region_obligations not empty: {:#?}",
+                inner.region_obligations
+            );
+            inner
+                .region_constraint_storage
+                .take()
+                .expect("regions already resolved")
+                .with_log(&mut inner.undo_log)
+                .into_infos_and_data()
+        };
+
+        let region_rels = &RegionRelations::new(self.tcx, outlives_env.free_region_map());
+
+        let (lexical_region_resolutions, errors) =
+            lexical_region_resolve::resolve(outlives_env.param_env, region_rels, var_infos, data);
+
+        let old_value = self.lexical_region_resolutions.replace(Some(lexical_region_resolutions));
+        assert!(old_value.is_none());
+
+        errors
+    }
+
+    /// Obtains (and clears) the current set of region
+    /// constraints. The inference context is still usable: further
+    /// unifications will simply add new constraints.
+    ///
+    /// This method is not meant to be used with normal lexical region
+    /// resolution. Rather, it is used in the NLL mode as a kind of
+    /// interim hack: basically we run normal type-check and generate
+    /// region constraints as normal, but then we take them and
+    /// translate them into the form that the NLL solver
+    /// understands. See the NLL module for mode details.
+    pub fn take_and_reset_region_constraints(&self) -> RegionConstraintData<'tcx> {
+        assert!(
+            self.inner.borrow().region_obligations.is_empty(),
+            "region_obligations not empty: {:#?}",
+            self.inner.borrow().region_obligations
+        );
+
+        self.inner.borrow_mut().unwrap_region_constraints().take_and_reset_data()
+    }
+
+    /// Gives temporary access to the region constraint data.
+    pub fn with_region_constraints<R>(
+        &self,
+        op: impl FnOnce(&RegionConstraintData<'tcx>) -> R,
+    ) -> R {
+        let mut inner = self.inner.borrow_mut();
+        op(inner.unwrap_region_constraints().data())
+    }
+}
diff --git a/compiler/rustc_infer/src/infer/outlives/obligations.rs b/compiler/rustc_infer/src/infer/outlives/obligations.rs
index bbe7d4c63..ccf11c61b 100644
--- a/compiler/rustc_infer/src/infer/outlives/obligations.rs
+++ b/compiler/rustc_infer/src/infer/outlives/obligations.rs
@@ -72,6 +72,8 @@ use rustc_middle::ty::subst::GenericArgKind;
 use rustc_middle::ty::{self, Region, SubstsRef, Ty, TyCtxt, TypeVisitableExt};
 use smallvec::smallvec;
 
+use super::env::OutlivesEnvironment;
+
 impl<'tcx> InferCtxt<'tcx> {
     /// Registers that the given region obligation must be resolved
     /// from within the scope of `body_id`. These regions are enqueued
@@ -112,39 +114,17 @@ impl<'tcx> InferCtxt<'tcx> {
         std::mem::take(&mut self.inner.borrow_mut().region_obligations)
     }
 
-    /// NOTE: Prefer using `TypeErrCtxt::check_region_obligations_and_report_errors`
-    /// instead of calling this directly.
-    ///
     /// Process the region obligations that must be proven (during
     /// `regionck`) for the given `body_id`, given information about
-    /// the region bounds in scope and so forth. This function must be
-    /// invoked for all relevant body-ids before region inference is
-    /// done (or else an assert will fire).
+    /// the region bounds in scope and so forth.
     ///
     /// See the `region_obligations` field of `InferCtxt` for some
     /// comments about how this function fits into the overall expected
     /// flow of the inferencer. The key point is that it is
     /// invoked after all type-inference variables have been bound --
-    /// towards the end of regionck. This also ensures that the
-    /// region-bound-pairs are available (see comments above regarding
-    /// closures).
-    ///
-    /// # Parameters
-    ///
-    /// - `region_bound_pairs_map`: the set of region bounds implied by
-    ///   the parameters and where-clauses. In particular, each pair
-    ///   `('a, K)` in this list tells us that the bounds in scope
-    ///   indicate that `K: 'a`, where `K` is either a generic
-    ///   parameter like `T` or a projection like `T::Item`.
-    /// - `param_env` is the parameter environment for the enclosing function.
-    /// - `body_id` is the body-id whose region obligations are being
-    ///   processed.
-    #[instrument(level = "debug", skip(self, region_bound_pairs))]
-    pub fn process_registered_region_obligations(
-        &self,
-        region_bound_pairs: &RegionBoundPairs<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) {
+    /// right before lexical region resolution.
+    #[instrument(level = "debug", skip(self, outlives_env))]
+    pub fn process_registered_region_obligations(&self, outlives_env: &OutlivesEnvironment<'tcx>) {
         assert!(
             !self.in_snapshot.get(),
             "cannot process registered region obligations in a snapshot"
@@ -153,15 +133,16 @@ impl<'tcx> InferCtxt<'tcx> {
         let my_region_obligations = self.take_registered_region_obligations();
 
         for RegionObligation { sup_type, sub_region, origin } in my_region_obligations {
-            debug!(
-                "process_registered_region_obligations: sup_type={:?} sub_region={:?} origin={:?}",
-                sup_type, sub_region, origin
-            );
-
+            debug!(?sup_type, ?sub_region, ?origin);
             let sup_type = self.resolve_vars_if_possible(sup_type);
 
-            let outlives =
-                &mut TypeOutlives::new(self, self.tcx, &region_bound_pairs, None, param_env);
+            let outlives = &mut TypeOutlives::new(
+                self,
+                self.tcx,
+                &outlives_env.region_bound_pairs(),
+                None,
+                outlives_env.param_env,
+            );
             let category = origin.to_constraint_category();
             outlives.type_must_outlive(origin, sup_type, sub_region, category);
         }
diff --git a/compiler/rustc_infer/src/infer/outlives/test_type_match.rs b/compiler/rustc_infer/src/infer/outlives/test_type_match.rs
index 3c6cc2b90..01f900f05 100644
--- a/compiler/rustc_infer/src/infer/outlives/test_type_match.rs
+++ b/compiler/rustc_infer/src/infer/outlives/test_type_match.rs
@@ -137,10 +137,6 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
         "Match"
     }
 
-    fn intercrate(&self) -> bool {
-        false
-    }
-
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
@@ -151,10 +147,6 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
         true
     } // irrelevant
 
-    fn mark_ambiguous(&mut self) {
-        bug!()
-    }
-
     #[instrument(level = "trace", skip(self))]
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
diff --git a/compiler/rustc_infer/src/infer/projection.rs b/compiler/rustc_infer/src/infer/projection.rs
index f79504770..fa6529dfa 100644
--- a/compiler/rustc_infer/src/infer/projection.rs
+++ b/compiler/rustc_infer/src/infer/projection.rs
@@ -26,7 +26,7 @@ impl<'tcx> InferCtxt<'tcx> {
             // completely change the normalization routine with the new solver.
             //
             // The new solver correctly handles projection equality so this hack
-            // is not necessary. if re-enabled it should emit `PredicateKind::AliasEq`
+            // is not necessary. if re-enabled it should emit `PredicateKind::AliasRelate`
             // not `PredicateKind::Clause(Clause::Projection(..))` as in the new solver
             // `Projection` is used as `normalizes-to` which will fail for `<T as Trait>::Assoc eq ?0`.
             return projection_ty.to_ty(self.tcx);
diff --git a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs
index e413b2bb5..b8ba98fc0 100644
--- a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs
+++ b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs
@@ -290,9 +290,9 @@ impl<'me, 'tcx> LeakCheck<'me, 'tcx> {
     ) -> TypeError<'tcx> {
         debug!("error: placeholder={:?}, other_region={:?}", placeholder, other_region);
         if self.overly_polymorphic {
-            TypeError::RegionsOverlyPolymorphic(placeholder.name, other_region)
+            TypeError::RegionsOverlyPolymorphic(placeholder.bound.kind, other_region)
         } else {
-            TypeError::RegionsInsufficientlyPolymorphic(placeholder.name, other_region)
+            TypeError::RegionsInsufficientlyPolymorphic(placeholder.bound.kind, other_region)
         }
     }
 }
@@ -424,9 +424,6 @@ impl<'tcx> MiniGraph<'tcx> {
                 &AddConstraint(Constraint::RegSubReg(a, b)) => {
                     each_edge(a, b);
                 }
-                &AddGiven(a, b) => {
-                    each_edge(a, tcx.mk_re_var(b));
-                }
                 &AddVerify(i) => span_bug!(
                     verifys[i].origin.span(),
                     "we never add verifications while doing higher-ranked things",
diff --git a/compiler/rustc_infer/src/infer/region_constraints/mod.rs b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
index 872f61747..7b272dfd2 100644
--- a/compiler/rustc_infer/src/infer/region_constraints/mod.rs
+++ b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
@@ -7,7 +7,7 @@ use super::{
     InferCtxtUndoLogs, MiscVariable, RegionVariableOrigin, Rollback, Snapshot, SubregionOrigin,
 };
 
-use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
+use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::intern::Interned;
 use rustc_data_structures::sync::Lrc;
 use rustc_data_structures::undo_log::UndoLogs;
@@ -104,26 +104,6 @@ pub struct RegionConstraintData<'tcx> {
     /// An example is a `A <= B` where neither `A` nor `B` are
     /// inference variables.
     pub verifys: Vec<Verify<'tcx>>,
-
-    /// A "given" is a relationship that is known to hold. In
-    /// particular, we often know from closure fn signatures that a
-    /// particular free region must be a subregion of a region
-    /// variable:
-    ///
-    ///    foo.iter().filter(<'a> |x: &'a &'b T| ...)
-    ///
-    /// In situations like this, `'b` is in fact a region variable
-    /// introduced by the call to `iter()`, and `'a` is a bound region
-    /// on the closure (as indicated by the `<'a>` prefix). If we are
-    /// naive, we wind up inferring that `'b` must be `'static`,
-    /// because we require that it be greater than `'a` and we do not
-    /// know what `'a` is precisely.
-    ///
-    /// This hashmap is used to avoid that naive scenario. Basically
-    /// we record the fact that `'a <= 'b` is implied by the fn
-    /// signature, and then ignore the constraint when solving
-    /// equations. This is a bit of a hack but seems to work.
-    pub givens: FxIndexSet<(Region<'tcx>, ty::RegionVid)>,
 }
 
 /// Represents a constraint that influences the inference process.
@@ -297,9 +277,6 @@ pub(crate) enum UndoLog<'tcx> {
     /// We added the given `verify`.
     AddVerify(usize),
 
-    /// We added the given `given`.
-    AddGiven(Region<'tcx>, ty::RegionVid),
-
     /// We added a GLB/LUB "combination variable".
     AddCombination(CombineMapType, TwoRegions<'tcx>),
 }
@@ -348,9 +325,6 @@ impl<'tcx> RegionConstraintStorage<'tcx> {
                 self.data.verifys.pop();
                 assert_eq!(self.data.verifys.len(), index);
             }
-            AddGiven(sub, sup) => {
-                self.data.givens.remove(&(sub, sup));
-            }
             AddCombination(Glb, ref regions) => {
                 self.glbs.remove(regions);
             }
@@ -420,7 +394,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
         // `RegionConstraintData` contains the relationship here.
         if *any_unifications {
             *any_unifications = false;
-            self.unification_table().reset_unifications(|_| UnifiedRegion(None));
+            self.unification_table_mut().reset_unifications(|_| UnifiedRegion::new(None));
         }
 
         data
@@ -447,7 +421,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
     ) -> RegionVid {
         let vid = self.var_infos.push(RegionVariableInfo { origin, universe });
 
-        let u_vid = self.unification_table().new_key(UnifiedRegion(None));
+        let u_vid = self.unification_table_mut().new_key(UnifiedRegion::new(None));
         assert_eq!(vid, u_vid.vid);
         self.undo_log.push(AddVar(vid));
         debug!("created new region variable {:?} in {:?} with origin {:?}", vid, universe, origin);
@@ -492,15 +466,6 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
         self.undo_log.push(AddVerify(index));
     }
 
-    pub(super) fn add_given(&mut self, sub: Region<'tcx>, sup: ty::RegionVid) {
-        // cannot add givens once regions are resolved
-        if self.data.givens.insert((sub, sup)) {
-            debug!("add_given({:?} <= {:?})", sub, sup);
-
-            self.undo_log.push(AddGiven(sub, sup));
-        }
-    }
-
     pub(super) fn make_eqregion(
         &mut self,
         origin: SubregionOrigin<'tcx>,
@@ -516,13 +481,13 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
             match (sub, sup) {
                 (Region(Interned(ReVar(sub), _)), Region(Interned(ReVar(sup), _))) => {
                     debug!("make_eqregion: unifying {:?} with {:?}", sub, sup);
-                    self.unification_table().union(*sub, *sup);
+                    self.unification_table_mut().union(*sub, *sup);
                     self.any_unifications = true;
                 }
                 (Region(Interned(ReVar(vid), _)), value)
                 | (value, Region(Interned(ReVar(vid), _))) => {
                     debug!("make_eqregion: unifying {:?} with {:?}", vid, value);
-                    self.unification_table().union_value(*vid, UnifiedRegion(Some(value)));
+                    self.unification_table_mut().union_value(*vid, UnifiedRegion::new(Some(value)));
                     self.any_unifications = true;
                 }
                 (_, _) => {}
@@ -633,28 +598,25 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
         }
     }
 
-    /// Resolves the passed RegionVid to the root RegionVid in the unification table
-    pub(super) fn opportunistic_resolve_var(&mut self, rid: ty::RegionVid) -> ty::RegionVid {
-        self.unification_table().find(rid).vid
-    }
-
-    /// If the Region is a `ReVar`, then resolves it either to the root value in
-    /// the unification table, if it exists, or to the root `ReVar` in the table.
-    /// If the Region is not a `ReVar`, just returns the Region itself.
-    pub fn opportunistic_resolve_region(
+    /// Resolves a region var to its value in the unification table, if it exists.
+    /// Otherwise, it is resolved to the root `ReVar` in the table.
+    pub fn opportunistic_resolve_var(
         &mut self,
         tcx: TyCtxt<'tcx>,
-        region: ty::Region<'tcx>,
+        vid: ty::RegionVid,
     ) -> ty::Region<'tcx> {
-        match *region {
-            ty::ReVar(rid) => {
-                let unified_region = self.unification_table().probe_value(rid);
-                unified_region.0.unwrap_or_else(|| {
-                    let root = self.unification_table().find(rid).vid;
-                    tcx.mk_re_var(root)
-                })
-            }
-            _ => region,
+        let mut ut = self.unification_table_mut(); // FIXME(rust-lang/ena#42): unnecessary mut
+        let root_vid = ut.find(vid).vid;
+        let resolved = ut
+            .probe_value(root_vid)
+            .get_value_ignoring_universes()
+            .unwrap_or_else(|| tcx.mk_re_var(root_vid));
+
+        // Don't resolve a variable to a region that it cannot name.
+        if self.var_universe(vid).can_name(self.universe(resolved)) {
+            resolved
+        } else {
+            tcx.mk_re_var(vid)
         }
     }
 
@@ -733,7 +695,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
     }
 
     #[inline]
-    fn unification_table(&mut self) -> super::UnificationTable<'_, 'tcx, RegionVidKey<'tcx>> {
+    fn unification_table_mut(&mut self) -> super::UnificationTable<'_, 'tcx, RegionVidKey<'tcx>> {
         ut::UnificationTable::with_log(&mut self.storage.unification_table, self.undo_log)
     }
 }
@@ -807,11 +769,8 @@ impl<'tcx> RegionConstraintData<'tcx> {
     /// Returns `true` if this region constraint data contains no constraints, and `false`
     /// otherwise.
     pub fn is_empty(&self) -> bool {
-        let RegionConstraintData { constraints, member_constraints, verifys, givens } = self;
-        constraints.is_empty()
-            && member_constraints.is_empty()
-            && verifys.is_empty()
-            && givens.is_empty()
+        let RegionConstraintData { constraints, member_constraints, verifys } = self;
+        constraints.is_empty() && member_constraints.is_empty() && verifys.is_empty()
     }
 }
 
diff --git a/compiler/rustc_infer/src/infer/resolve.rs b/compiler/rustc_infer/src/infer/resolve.rs
index 5bb358329..4f49f4165 100644
--- a/compiler/rustc_infer/src/infer/resolve.rs
+++ b/compiler/rustc_infer/src/infer/resolve.rs
@@ -85,15 +85,12 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for OpportunisticRegionResolver<'a, 'tcx
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
-            ty::ReVar(rid) => {
-                let resolved = self
-                    .infcx
-                    .inner
-                    .borrow_mut()
-                    .unwrap_region_constraints()
-                    .opportunistic_resolve_var(rid);
-                TypeFolder::interner(self).mk_re_var(resolved)
-            }
+            ty::ReVar(vid) => self
+                .infcx
+                .inner
+                .borrow_mut()
+                .unwrap_region_constraints()
+                .opportunistic_resolve_var(TypeFolder::interner(self), vid),
             _ => r,
         }
     }
diff --git a/compiler/rustc_infer/src/infer/sub.rs b/compiler/rustc_infer/src/infer/sub.rs
index 3e8c2052d..0dd73a6e9 100644
--- a/compiler/rustc_infer/src/infer/sub.rs
+++ b/compiler/rustc_infer/src/infer/sub.rs
@@ -1,5 +1,5 @@
 use super::combine::{CombineFields, RelationDir};
-use super::{ObligationEmittingRelation, SubregionOrigin};
+use super::{DefineOpaqueTypes, ObligationEmittingRelation, SubregionOrigin};
 
 use crate::traits::{Obligation, PredicateObligations};
 use rustc_middle::ty::relate::{Cause, Relate, RelateResult, TypeRelation};
@@ -35,10 +35,6 @@ impl<'tcx> TypeRelation<'tcx> for Sub<'_, '_, 'tcx> {
         "Sub"
     }
 
-    fn intercrate(&self) -> bool {
-        self.fields.infcx.intercrate
-    }
-
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.fields.infcx.tcx
     }
@@ -51,10 +47,6 @@ impl<'tcx> TypeRelation<'tcx> for Sub<'_, '_, 'tcx> {
         self.a_is_expected
     }
 
-    fn mark_ambiguous(&mut self) {
-        self.fields.mark_ambiguous()
-    }
-
     fn with_cause<F, R>(&mut self, cause: Cause, f: F) -> R
     where
         F: FnOnce(&mut Self) -> R,
@@ -138,7 +130,8 @@ impl<'tcx> TypeRelation<'tcx> for Sub<'_, '_, 'tcx> {
             }
             (&ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }), _)
             | (_, &ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }))
-                if self.fields.define_opaque_types && def_id.is_local() =>
+                if self.fields.define_opaque_types == DefineOpaqueTypes::Yes
+                    && def_id.is_local() =>
             {
                 self.fields.obligations.extend(
                     infcx
@@ -235,4 +228,8 @@ impl<'tcx> ObligationEmittingRelation<'tcx> for Sub<'_, '_, 'tcx> {
     fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
         self.fields.register_obligations(obligations);
     }
+
+    fn alias_relate_direction(&self) -> ty::AliasRelationDirection {
+        ty::AliasRelationDirection::Subtype
+    }
 }
diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs
index 263c6a47d..f7ab05b2d 100644
--- a/compiler/rustc_infer/src/infer/type_variable.rs
+++ b/compiler/rustc_infer/src/infer/type_variable.rs
@@ -190,6 +190,11 @@ impl<'tcx> TypeVariableStorage<'tcx> {
     ) -> TypeVariableTable<'a, 'tcx> {
         TypeVariableTable { storage: self, undo_log }
     }
+
+    #[inline]
+    pub(crate) fn eq_relations_ref(&self) -> &ut::UnificationTableStorage<TyVidEqKey<'tcx>> {
+        &self.eq_relations
+    }
 }
 
 impl<'tcx> TypeVariableTable<'_, 'tcx> {
diff --git a/compiler/rustc_infer/src/lib.rs b/compiler/rustc_infer/src/lib.rs
index bdc313c21..738a12376 100644
--- a/compiler/rustc_infer/src/lib.rs
+++ b/compiler/rustc_infer/src/lib.rs
@@ -41,4 +41,4 @@ mod errors;
 pub mod infer;
 pub mod traits;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_infer/src/traits/engine.rs b/compiler/rustc_infer/src/traits/engine.rs
index f75344f20..b8940e2f0 100644
--- a/compiler/rustc_infer/src/traits/engine.rs
+++ b/compiler/rustc_infer/src/traits/engine.rs
@@ -36,9 +36,10 @@ pub trait TraitEngine<'tcx>: 'tcx {
         obligation: PredicateObligation<'tcx>,
     );
 
+    #[must_use]
     fn select_where_possible(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>>;
 
-    fn collect_remaining_errors(&mut self) -> Vec<FulfillmentError<'tcx>>;
+    fn collect_remaining_errors(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>>;
 
     fn pending_obligations(&self) -> Vec<PredicateObligation<'tcx>>;
 
@@ -58,6 +59,7 @@ pub trait TraitEngineExt<'tcx> {
         obligations: impl IntoIterator<Item = PredicateObligation<'tcx>>,
     );
 
+    #[must_use]
     fn select_all_or_error(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>>;
 }
 
@@ -78,6 +80,6 @@ impl<'tcx, T: ?Sized + TraitEngine<'tcx>> TraitEngineExt<'tcx> for T {
             return errors;
         }
 
-        self.collect_remaining_errors()
+        self.collect_remaining_errors(infcx)
     }
 }
diff --git a/compiler/rustc_infer/src/traits/mod.rs b/compiler/rustc_infer/src/traits/mod.rs
index 3a8289966..e01b6caf4 100644
--- a/compiler/rustc_infer/src/traits/mod.rs
+++ b/compiler/rustc_infer/src/traits/mod.rs
@@ -8,6 +8,8 @@ mod project;
 mod structural_impls;
 pub mod util;
 
+use std::cmp;
+
 use hir::def_id::LocalDefId;
 use rustc_hir as hir;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
@@ -53,6 +55,12 @@ pub struct Obligation<'tcx, T> {
     pub recursion_depth: usize,
 }
 
+impl<'tcx, P> From<Obligation<'tcx, P>> for solve::Goal<'tcx, P> {
+    fn from(value: Obligation<'tcx, P>) -> Self {
+        solve::Goal { param_env: value.param_env, predicate: value.predicate }
+    }
+}
+
 pub type PredicateObligation<'tcx> = Obligation<'tcx, ty::Predicate<'tcx>>;
 pub type TraitObligation<'tcx> = Obligation<'tcx, ty::PolyTraitPredicate<'tcx>>;
 
@@ -120,7 +128,11 @@ pub enum FulfillmentErrorCode<'tcx> {
     CodeProjectionError(MismatchedProjectionTypes<'tcx>),
     CodeSubtypeError(ExpectedFound<Ty<'tcx>>, TypeError<'tcx>), // always comes from a SubtypePredicate
     CodeConstEquateError(ExpectedFound<Const<'tcx>>, TypeError<'tcx>),
-    CodeAmbiguity,
+    CodeAmbiguity {
+        /// Overflow reported from the new solver `-Ztrait-solver=next`, which will
+        /// be reported as an regular error as opposed to a fatal error.
+        overflow: bool,
+    },
 }
 
 impl<'tcx, O> Obligation<'tcx, O> {
@@ -133,6 +145,14 @@ impl<'tcx, O> Obligation<'tcx, O> {
         Self::with_depth(tcx, cause, 0, param_env, predicate)
     }
 
+    /// We often create nested obligations without setting the correct depth.
+    ///
+    /// To deal with this evaluate and fulfill explicitly update the depth
+    /// of nested obligations using this function.
+    pub fn set_depth_from_parent(&mut self, parent_depth: usize) {
+        self.recursion_depth = cmp::max(parent_depth + 1, self.recursion_depth);
+    }
+
     pub fn with_depth(
         tcx: TyCtxt<'tcx>,
         cause: ObligationCause<'tcx>,
diff --git a/compiler/rustc_infer/src/traits/structural_impls.rs b/compiler/rustc_infer/src/traits/structural_impls.rs
index 3a5273b03..1563d92af 100644
--- a/compiler/rustc_infer/src/traits/structural_impls.rs
+++ b/compiler/rustc_infer/src/traits/structural_impls.rs
@@ -46,7 +46,8 @@ impl<'tcx> fmt::Debug for traits::FulfillmentErrorCode<'tcx> {
             super::CodeConstEquateError(ref a, ref b) => {
                 write!(f, "CodeConstEquateError({:?}, {:?})", a, b)
             }
-            super::CodeAmbiguity => write!(f, "Ambiguity"),
+            super::CodeAmbiguity { overflow: false } => write!(f, "Ambiguity"),
+            super::CodeAmbiguity { overflow: true } => write!(f, "Overflow"),
             super::CodeCycle(ref cycle) => write!(f, "Cycle({:?})", cycle),
         }
     }
diff --git a/compiler/rustc_infer/src/traits/util.rs b/compiler/rustc_infer/src/traits/util.rs
index c07ff5165..ef01d5d51 100644
--- a/compiler/rustc_infer/src/traits/util.rs
+++ b/compiler/rustc_infer/src/traits/util.rs
@@ -1,7 +1,7 @@
 use smallvec::smallvec;
 
 use crate::infer::outlives::components::{push_outlives_components, Component};
-use crate::traits::{self, Obligation, ObligationCause, PredicateObligation};
+use crate::traits::{self, Obligation, PredicateObligation};
 use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
 use rustc_middle::ty::{self, ToPredicate, TyCtxt};
 use rustc_span::symbol::Ident;
@@ -66,93 +66,146 @@ impl<'tcx> Extend<ty::Predicate<'tcx>> for PredicateSet<'tcx> {
 /// if we know that `T: Ord`, the elaborator would deduce that `T: PartialOrd`
 /// holds as well. Similarly, if we have `trait Foo: 'static`, and we know that
 /// `T: Foo`, then we know that `T: 'static`.
-pub struct Elaborator<'tcx> {
-    stack: Vec<PredicateObligation<'tcx>>,
+pub struct Elaborator<'tcx, O> {
+    stack: Vec<O>,
     visited: PredicateSet<'tcx>,
+    only_self: bool,
 }
 
-pub fn elaborate_trait_ref<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    trait_ref: ty::PolyTraitRef<'tcx>,
-) -> Elaborator<'tcx> {
-    elaborate_predicates(tcx, std::iter::once(trait_ref.without_const().to_predicate(tcx)))
+/// Describes how to elaborate an obligation into a sub-obligation.
+///
+/// For [`Obligation`], a sub-obligation is combined with the current obligation's
+/// param-env and cause code. For [`ty::Predicate`], none of this is needed, since
+/// there is no param-env or cause code to copy over.
+pub trait Elaboratable<'tcx> {
+    fn predicate(&self) -> ty::Predicate<'tcx>;
+
+    // Makes a new `Self` but with a different predicate.
+    fn child(&self, predicate: ty::Predicate<'tcx>) -> Self;
+
+    // Makes a new `Self` but with a different predicate and a different cause
+    // code (if `Self` has one).
+    fn child_with_derived_cause(
+        &self,
+        predicate: ty::Predicate<'tcx>,
+        span: Span,
+        parent_trait_pred: ty::PolyTraitPredicate<'tcx>,
+        index: usize,
+    ) -> Self;
 }
 
-pub fn elaborate_trait_refs<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    trait_refs: impl Iterator<Item = ty::PolyTraitRef<'tcx>>,
-) -> Elaborator<'tcx> {
-    let predicates = trait_refs.map(|trait_ref| trait_ref.without_const().to_predicate(tcx));
-    elaborate_predicates(tcx, predicates)
+impl<'tcx> Elaboratable<'tcx> for PredicateObligation<'tcx> {
+    fn predicate(&self) -> ty::Predicate<'tcx> {
+        self.predicate
+    }
+
+    fn child(&self, predicate: ty::Predicate<'tcx>) -> Self {
+        Obligation {
+            cause: self.cause.clone(),
+            param_env: self.param_env,
+            recursion_depth: 0,
+            predicate,
+        }
+    }
+
+    fn child_with_derived_cause(
+        &self,
+        predicate: ty::Predicate<'tcx>,
+        span: Span,
+        parent_trait_pred: ty::PolyTraitPredicate<'tcx>,
+        index: usize,
+    ) -> Self {
+        let cause = self.cause.clone().derived_cause(parent_trait_pred, |derived| {
+            traits::ImplDerivedObligation(Box::new(traits::ImplDerivedObligationCause {
+                derived,
+                impl_or_alias_def_id: parent_trait_pred.def_id(),
+                impl_def_predicate_index: Some(index),
+                span,
+            }))
+        });
+        Obligation { cause, param_env: self.param_env, recursion_depth: 0, predicate }
+    }
 }
 
-pub fn elaborate_predicates<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    predicates: impl Iterator<Item = ty::Predicate<'tcx>>,
-) -> Elaborator<'tcx> {
-    let obligations = predicates
-        .map(|predicate| {
-            predicate_obligation(predicate, ty::ParamEnv::empty(), ObligationCause::dummy())
-        })
-        .collect();
-    elaborate_obligations(tcx, obligations)
+impl<'tcx> Elaboratable<'tcx> for ty::Predicate<'tcx> {
+    fn predicate(&self) -> ty::Predicate<'tcx> {
+        *self
+    }
+
+    fn child(&self, predicate: ty::Predicate<'tcx>) -> Self {
+        predicate
+    }
+
+    fn child_with_derived_cause(
+        &self,
+        predicate: ty::Predicate<'tcx>,
+        _span: Span,
+        _parent_trait_pred: ty::PolyTraitPredicate<'tcx>,
+        _index: usize,
+    ) -> Self {
+        predicate
+    }
 }
 
-pub fn elaborate_predicates_with_span<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    predicates: impl Iterator<Item = (ty::Predicate<'tcx>, Span)>,
-) -> Elaborator<'tcx> {
-    let obligations = predicates
-        .map(|(predicate, span)| {
-            predicate_obligation(
-                predicate,
-                ty::ParamEnv::empty(),
-                ObligationCause::dummy_with_span(span),
-            )
-        })
-        .collect();
-    elaborate_obligations(tcx, obligations)
+impl<'tcx> Elaboratable<'tcx> for (ty::Predicate<'tcx>, Span) {
+    fn predicate(&self) -> ty::Predicate<'tcx> {
+        self.0
+    }
+
+    fn child(&self, predicate: ty::Predicate<'tcx>) -> Self {
+        (predicate, self.1)
+    }
+
+    fn child_with_derived_cause(
+        &self,
+        predicate: ty::Predicate<'tcx>,
+        _span: Span,
+        _parent_trait_pred: ty::PolyTraitPredicate<'tcx>,
+        _index: usize,
+    ) -> Self {
+        (predicate, self.1)
+    }
 }
 
-pub fn elaborate_obligations<'tcx>(
+pub fn elaborate<'tcx, O: Elaboratable<'tcx>>(
     tcx: TyCtxt<'tcx>,
-    obligations: Vec<PredicateObligation<'tcx>>,
-) -> Elaborator<'tcx> {
-    let mut elaborator = Elaborator { stack: Vec::new(), visited: PredicateSet::new(tcx) };
+    obligations: impl IntoIterator<Item = O>,
+) -> Elaborator<'tcx, O> {
+    let mut elaborator =
+        Elaborator { stack: Vec::new(), visited: PredicateSet::new(tcx), only_self: false };
     elaborator.extend_deduped(obligations);
     elaborator
 }
 
-fn predicate_obligation<'tcx>(
-    predicate: ty::Predicate<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
-    cause: ObligationCause<'tcx>,
-) -> PredicateObligation<'tcx> {
-    Obligation { cause, param_env, recursion_depth: 0, predicate }
-}
-
-impl<'tcx> Elaborator<'tcx> {
-    fn extend_deduped(&mut self, obligations: impl IntoIterator<Item = PredicateObligation<'tcx>>) {
+impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
+    fn extend_deduped(&mut self, obligations: impl IntoIterator<Item = O>) {
         // Only keep those bounds that we haven't already seen.
         // This is necessary to prevent infinite recursion in some
         // cases. One common case is when people define
         // `trait Sized: Sized { }` rather than `trait Sized { }`.
         // let visited = &mut self.visited;
-        self.stack.extend(obligations.into_iter().filter(|o| self.visited.insert(o.predicate)));
+        self.stack.extend(obligations.into_iter().filter(|o| self.visited.insert(o.predicate())));
     }
 
-    pub fn filter_to_traits(self) -> FilterToTraits<Self> {
-        FilterToTraits::new(self)
+    /// Filter to only the supertraits of trait predicates, i.e. only the predicates
+    /// that have `Self` as their self type, instead of all implied predicates.
+    pub fn filter_only_self(mut self) -> Self {
+        self.only_self = true;
+        self
     }
 
-    fn elaborate(&mut self, obligation: &PredicateObligation<'tcx>) {
+    fn elaborate(&mut self, elaboratable: &O) {
         let tcx = self.visited.tcx;
 
-        let bound_predicate = obligation.predicate.kind();
+        let bound_predicate = elaboratable.predicate().kind();
         match bound_predicate.skip_binder() {
             ty::PredicateKind::Clause(ty::Clause::Trait(data)) => {
-                // Get predicates declared on the trait.
-                let predicates = tcx.super_predicates_of(data.def_id());
+                // Get predicates implied by the trait, or only super predicates if we only care about self predicates.
+                let predicates = if self.only_self {
+                    tcx.super_predicates_of(data.def_id())
+                } else {
+                    tcx.implied_predicates_of(data.def_id())
+                };
 
                 let obligations =
                     predicates.predicates.iter().enumerate().map(|(index, &(mut pred, span))| {
@@ -160,24 +213,11 @@ impl<'tcx> Elaborator<'tcx> {
                         if data.constness == ty::BoundConstness::NotConst {
                             pred = pred.without_const(tcx);
                         }
-
-                        let cause = obligation.cause.clone().derived_cause(
-                            bound_predicate.rebind(data),
-                            |derived| {
-                                traits::ImplDerivedObligation(Box::new(
-                                    traits::ImplDerivedObligationCause {
-                                        derived,
-                                        impl_or_alias_def_id: data.def_id(),
-                                        impl_def_predicate_index: Some(index),
-                                        span,
-                                    },
-                                ))
-                            },
-                        );
-                        predicate_obligation(
+                        elaboratable.child_with_derived_cause(
                             pred.subst_supertrait(tcx, &bound_predicate.rebind(data.trait_ref)),
-                            obligation.param_env,
-                            cause,
+                            span,
+                            bound_predicate.rebind(data),
+                            index,
                         )
                     });
                 debug!(?data, ?obligations, "super_predicates");
@@ -280,20 +320,14 @@ impl<'tcx> Elaborator<'tcx> {
                         .map(|predicate_kind| {
                             bound_predicate.rebind(predicate_kind).to_predicate(tcx)
                         })
-                        .map(|predicate| {
-                            predicate_obligation(
-                                predicate,
-                                obligation.param_env,
-                                obligation.cause.clone(),
-                            )
-                        }),
+                        .map(|predicate| elaboratable.child(predicate)),
                 );
             }
             ty::PredicateKind::TypeWellFormedFromEnv(..) => {
                 // Nothing to elaborate
             }
             ty::PredicateKind::Ambiguous => {}
-            ty::PredicateKind::AliasEq(..) => {
+            ty::PredicateKind::AliasRelate(..) => {
                 // No
             }
             ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..)) => {
@@ -303,8 +337,8 @@ impl<'tcx> Elaborator<'tcx> {
     }
 }
 
-impl<'tcx> Iterator for Elaborator<'tcx> {
-    type Item = PredicateObligation<'tcx>;
+impl<'tcx, O: Elaboratable<'tcx>> Iterator for Elaborator<'tcx, O> {
+    type Item = O;
 
     fn size_hint(&self) -> (usize, Option<usize>) {
         (self.stack.len(), None)
@@ -325,23 +359,23 @@ impl<'tcx> Iterator for Elaborator<'tcx> {
 // Supertrait iterator
 ///////////////////////////////////////////////////////////////////////////
 
-pub type Supertraits<'tcx> = FilterToTraits<Elaborator<'tcx>>;
-
 pub fn supertraits<'tcx>(
     tcx: TyCtxt<'tcx>,
     trait_ref: ty::PolyTraitRef<'tcx>,
-) -> Supertraits<'tcx> {
-    elaborate_trait_ref(tcx, trait_ref).filter_to_traits()
+) -> impl Iterator<Item = ty::PolyTraitRef<'tcx>> {
+    elaborate(tcx, [trait_ref.to_predicate(tcx)]).filter_only_self().filter_to_traits()
 }
 
 pub fn transitive_bounds<'tcx>(
     tcx: TyCtxt<'tcx>,
-    bounds: impl Iterator<Item = ty::PolyTraitRef<'tcx>>,
-) -> Supertraits<'tcx> {
-    elaborate_trait_refs(tcx, bounds).filter_to_traits()
+    trait_refs: impl Iterator<Item = ty::PolyTraitRef<'tcx>>,
+) -> impl Iterator<Item = ty::PolyTraitRef<'tcx>> {
+    elaborate(tcx, trait_refs.map(|trait_ref| trait_ref.to_predicate(tcx)))
+        .filter_only_self()
+        .filter_to_traits()
 }
 
-/// A specialized variant of `elaborate_trait_refs` that only elaborates trait references that may
+/// A specialized variant of `elaborate` that only elaborates trait references that may
 /// define the given associated type `assoc_name`. It uses the
 /// `super_predicates_that_define_assoc_type` query to avoid enumerating super-predicates that
 /// aren't related to `assoc_item`. This is used when resolving types like `Self::Item` or
@@ -358,10 +392,8 @@ pub fn transitive_bounds_that_define_assoc_type<'tcx>(
         while let Some(trait_ref) = stack.pop() {
             let anon_trait_ref = tcx.anonymize_bound_vars(trait_ref);
             if visited.insert(anon_trait_ref) {
-                let super_predicates = tcx.super_predicates_that_define_assoc_type((
-                    trait_ref.def_id(),
-                    Some(assoc_name),
-                ));
+                let super_predicates =
+                    tcx.super_predicates_that_define_assoc_type((trait_ref.def_id(), assoc_name));
                 for (super_predicate, _) in super_predicates.predicates {
                     let subst_predicate = super_predicate.subst_supertrait(tcx, &trait_ref);
                     if let Some(binder) = subst_predicate.to_opt_poly_trait_pred() {
@@ -381,24 +413,24 @@ pub fn transitive_bounds_that_define_assoc_type<'tcx>(
 // Other
 ///////////////////////////////////////////////////////////////////////////
 
+impl<'tcx> Elaborator<'tcx, ty::Predicate<'tcx>> {
+    fn filter_to_traits(self) -> FilterToTraits<Self> {
+        FilterToTraits { base_iterator: self }
+    }
+}
+
 /// A filter around an iterator of predicates that makes it yield up
 /// just trait references.
 pub struct FilterToTraits<I> {
     base_iterator: I,
 }
 
-impl<I> FilterToTraits<I> {
-    fn new(base: I) -> FilterToTraits<I> {
-        FilterToTraits { base_iterator: base }
-    }
-}
-
-impl<'tcx, I: Iterator<Item = PredicateObligation<'tcx>>> Iterator for FilterToTraits<I> {
+impl<'tcx, I: Iterator<Item = ty::Predicate<'tcx>>> Iterator for FilterToTraits<I> {
     type Item = ty::PolyTraitRef<'tcx>;
 
     fn next(&mut self) -> Option<ty::PolyTraitRef<'tcx>> {
-        while let Some(obligation) = self.base_iterator.next() {
-            if let Some(data) = obligation.predicate.to_opt_poly_trait_pred() {
+        while let Some(pred) = self.base_iterator.next() {
+            if let Some(data) = pred.to_opt_poly_trait_pred() {
                 return Some(data.map_bound(|t| t.trait_ref));
             }
         }
diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml
index ac6e8fca6..98d3ab87f 100644
--- a/compiler/rustc_interface/Cargo.toml
+++ b/compiler/rustc_interface/Cargo.toml
@@ -8,14 +8,15 @@ edition = "2021"
 [dependencies]
 libloading = "0.7.1"
 tracing = "0.1"
-rustc-rayon-core = { version = "0.4.0", optional = true }
-rayon = { version = "0.4.0", package = "rustc-rayon", optional = true }
+rustc-rayon-core = { version = "0.5.0", optional = true }
+rustc-rayon = { version = "0.5.0", optional = true }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_borrowck = { path = "../rustc_borrowck" }
 rustc_builtin_macros = { path = "../rustc_builtin_macros" }
 rustc_expand = { path = "../rustc_expand" }
+rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_session = { path = "../rustc_session" }
@@ -51,4 +52,4 @@ rustc_ty_utils = { path = "../rustc_ty_utils" }
 
 [features]
 llvm = ['rustc_codegen_llvm']
-rustc_use_parallel_compiler = ['rayon', 'rustc-rayon-core', 'rustc_query_impl/rustc_use_parallel_compiler', 'rustc_errors/rustc_use_parallel_compiler']
+rustc_use_parallel_compiler = ['rustc-rayon', 'rustc-rayon-core', 'rustc_query_impl/rustc_use_parallel_compiler', 'rustc_errors/rustc_use_parallel_compiler']
diff --git a/compiler/rustc_interface/locales/en-US.ftl b/compiler/rustc_interface/messages.ftl
index 37994899a..37994899a 100644
--- a/compiler/rustc_interface/locales/en-US.ftl
+++ b/compiler/rustc_interface/messages.ftl
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 5e38ca034..be7fa9378 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -292,7 +292,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
                 override_queries: config.override_queries,
             };
 
-            rustc_span::with_source_map(compiler.sess.parse_sess.clone_source_map(), move || {
+            rustc_span::set_source_map(compiler.sess.parse_sess.clone_source_map(), move || {
                 let r = {
                     let _sess_abort_error = OnDrop(|| {
                         compiler.sess.finish_diagnostics(registry);
diff --git a/compiler/rustc_interface/src/lib.rs b/compiler/rustc_interface/src/lib.rs
index 1abbe8d4f..9664ba8bd 100644
--- a/compiler/rustc_interface/src/lib.rs
+++ b/compiler/rustc_interface/src/lib.rs
@@ -2,7 +2,7 @@
 #![feature(decl_macro)]
 #![feature(internal_output_capture)]
 #![feature(thread_spawn_unchecked)]
-#![feature(once_cell)]
+#![feature(lazy_cell)]
 #![feature(try_blocks)]
 #![recursion_limit = "256"]
 #![allow(rustc::potential_query_instability)]
@@ -31,4 +31,4 @@ pub use queries::Queries;
 #[cfg(test)]
 mod tests;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 81c1d665e..0e4e20c7c 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -3,7 +3,6 @@ use crate::interface::{Compiler, Result};
 use crate::proc_macro_decls;
 use crate::util;
 
-use ast::CRATE_NODE_ID;
 use rustc_ast::{self as ast, visit};
 use rustc_borrowck as mir_borrowck;
 use rustc_codegen_ssa::traits::CodegenBackend;
@@ -11,7 +10,8 @@ use rustc_data_structures::parallel;
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal};
 use rustc_errors::PResult;
-use rustc_expand::base::{ExtCtxt, LintStoreExpand, ResolverExpand};
+use rustc_expand::base::{ExtCtxt, LintStoreExpand};
+use rustc_fs_util::try_canonicalize;
 use rustc_hir::def_id::{StableCrateId, LOCAL_CRATE};
 use rustc_lint::{unerased_lint_store, BufferedEarlyLint, EarlyCheckNode, LintStore};
 use rustc_metadata::creader::CStore;
@@ -26,7 +26,7 @@ use rustc_plugin_impl as plugin;
 use rustc_query_impl::{OnDiskCache, Queries as TcxQueries};
 use rustc_resolve::Resolver;
 use rustc_session::config::{CrateType, Input, OutputFilenames, OutputType};
-use rustc_session::cstore::{CrateStoreDyn, MetadataLoader, Untracked};
+use rustc_session::cstore::{MetadataLoader, Untracked};
 use rustc_session::output::filename_for_input;
 use rustc_session::search_paths::PathKind;
 use rustc_session::{Limit, Session};
@@ -76,22 +76,14 @@ pub fn register_plugins<'a>(
     sess: &'a Session,
     metadata_loader: &'a dyn MetadataLoader,
     register_lints: impl Fn(&Session, &mut LintStore),
-    mut krate: ast::Crate,
+    pre_configured_attrs: &[ast::Attribute],
     crate_name: Symbol,
-) -> Result<(ast::Crate, LintStore)> {
-    krate = sess.time("attributes_injection", || {
-        rustc_builtin_macros::cmdline_attrs::inject(
-            krate,
-            &sess.parse_sess,
-            &sess.opts.unstable_opts.crate_attr,
-        )
-    });
-
-    let (krate, features) = rustc_expand::config::features(sess, krate, CRATE_NODE_ID);
+) -> Result<LintStore> {
     // these need to be set "early" so that expansion sees `quote` if enabled.
+    let features = rustc_expand::config::features(sess, pre_configured_attrs);
     sess.init_features(features);
 
-    let crate_types = util::collect_crate_types(sess, &krate.attrs);
+    let crate_types = util::collect_crate_types(sess, pre_configured_attrs);
     sess.init_crate_types(crate_types);
 
     let stable_crate_id = StableCrateId::new(
@@ -117,8 +109,9 @@ pub fn register_plugins<'a>(
     let mut lint_store = rustc_lint::new_lint_store(sess.enable_internal_lints());
     register_lints(sess, &mut lint_store);
 
-    let registrars =
-        sess.time("plugin_loading", || plugin::load::load_plugins(sess, metadata_loader, &krate));
+    let registrars = sess.time("plugin_loading", || {
+        plugin::load::load_plugins(sess, metadata_loader, pre_configured_attrs)
+    });
     sess.time("plugin_registration", || {
         let mut registry = plugin::Registry { lint_store: &mut lint_store };
         for registrar in registrars {
@@ -126,7 +119,7 @@ pub fn register_plugins<'a>(
         }
     });
 
-    Ok((krate, lint_store))
+    Ok(lint_store)
 }
 
 fn pre_expansion_lint<'a>(
@@ -173,19 +166,29 @@ impl LintStoreExpand for LintStoreExpandImpl<'_> {
 /// harness if one is to be provided, injection of a dependency on the
 /// standard library and prelude, and name resolution.
 #[instrument(level = "trace", skip(krate, resolver))]
-fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>) -> ast::Crate {
+fn configure_and_expand(
+    mut krate: ast::Crate,
+    pre_configured_attrs: &[ast::Attribute],
+    resolver: &mut Resolver<'_, '_>,
+) -> ast::Crate {
     let tcx = resolver.tcx();
     let sess = tcx.sess;
     let lint_store = unerased_lint_store(tcx);
     let crate_name = tcx.crate_name(LOCAL_CRATE);
-    pre_expansion_lint(sess, lint_store, resolver.registered_tools(), &krate, crate_name);
+    let lint_check_node = (&krate, pre_configured_attrs);
+    pre_expansion_lint(sess, lint_store, tcx.registered_tools(()), lint_check_node, crate_name);
     rustc_builtin_macros::register_builtin_macros(resolver);
 
-    krate = sess.time("crate_injection", || {
-        rustc_builtin_macros::standard_library_imports::inject(krate, resolver, sess)
+    let num_standard_library_imports = sess.time("crate_injection", || {
+        rustc_builtin_macros::standard_library_imports::inject(
+            &mut krate,
+            pre_configured_attrs,
+            resolver,
+            sess,
+        )
     });
 
-    util::check_attr_crate_type(sess, &krate.attrs, &mut resolver.lint_buffer());
+    util::check_attr_crate_type(sess, pre_configured_attrs, &mut resolver.lint_buffer());
 
     // Expand all macros
     krate = sess.time("macro_expand_crate", || {
@@ -222,12 +225,12 @@ fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>)
 
         // Create the config for macro expansion
         let features = sess.features_untracked();
-        let recursion_limit = get_recursion_limit(&krate.attrs, sess);
+        let recursion_limit = get_recursion_limit(pre_configured_attrs, sess);
         let cfg = rustc_expand::expand::ExpansionConfig {
             features: Some(features),
             recursion_limit,
             trace_mac: sess.opts.unstable_opts.trace_macros,
-            should_test: sess.opts.test,
+            should_test: sess.is_test_crate(),
             span_debug: sess.opts.unstable_opts.span_debug,
             proc_macro_backtrace: sess.opts.unstable_opts.proc_macro_backtrace,
             ..rustc_expand::expand::ExpansionConfig::default(crate_name.to_string())
@@ -235,6 +238,7 @@ fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>)
 
         let lint_store = LintStoreExpandImpl(lint_store);
         let mut ecx = ExtCtxt::new(sess, cfg, resolver, Some(&lint_store));
+        ecx.num_standard_library_imports = num_standard_library_imports;
         // Expand macros now!
         let krate = sess.time("expand_crate", || ecx.monotonic_expander().expand_crate(krate));
 
@@ -263,7 +267,7 @@ fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>)
     });
 
     sess.time("maybe_building_test_harness", || {
-        rustc_builtin_macros::test_harness::inject(sess, resolver, &mut krate)
+        rustc_builtin_macros::test_harness::inject(&mut krate, sess, resolver)
     });
 
     let has_proc_macro_decls = sess.time("AST_validation", || {
@@ -287,12 +291,12 @@ fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>)
         sess.emit_warning(errors::ProcMacroCratePanicAbort);
     }
 
-    krate = sess.time("maybe_create_a_macro_crate", || {
-        let is_test_crate = sess.opts.test;
+    sess.time("maybe_create_a_macro_crate", || {
+        let is_test_crate = sess.is_test_crate();
         rustc_builtin_macros::proc_macro_harness::inject(
+            &mut krate,
             sess,
             resolver,
-            krate,
             is_proc_macro_crate,
             has_proc_macro_decls,
             is_test_crate,
@@ -302,6 +306,16 @@ fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>)
 
     // Done with macro expansion!
 
+    resolver.resolve_crate(&krate);
+
+    krate
+}
+
+fn early_lint_checks(tcx: TyCtxt<'_>, (): ()) {
+    let sess = tcx.sess;
+    let (resolver, krate) = &*tcx.resolver_for_lowering(()).borrow();
+    let mut lint_buffer = resolver.lint_buffer.steal();
+
     if sess.opts.unstable_opts.input_stats {
         eprintln!("Post-expansion node count: {}", count_nodes(&krate));
     }
@@ -310,8 +324,6 @@ fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>)
         hir_stats::print_ast_stats(&krate, "POST EXPANSION AST STATS", "ast-stats-2");
     }
 
-    resolver.resolve_crate(&krate);
-
     // Needs to go *after* expansion to be able to check the results of macro expansion.
     sess.time("complete_gated_feature_checking", || {
         rustc_ast_passes::feature_gate::check_crate(&krate, sess);
@@ -321,7 +333,7 @@ fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>)
     sess.parse_sess.buffered_lints.with_lock(|buffered_lints| {
         info!("{} parse sess buffered_lints", buffered_lints.len());
         for early_lint in buffered_lints.drain(..) {
-            resolver.lint_buffer().add_early_lint(early_lint);
+            lint_buffer.add_early_lint(early_lint);
         }
     });
 
@@ -340,20 +352,16 @@ fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>)
         }
     });
 
-    sess.time("early_lint_checks", || {
-        let lint_buffer = Some(std::mem::take(resolver.lint_buffer()));
-        rustc_lint::check_ast_node(
-            sess,
-            false,
-            lint_store,
-            resolver.registered_tools(),
-            lint_buffer,
-            rustc_lint::BuiltinCombinedEarlyLintPass::new(),
-            &krate,
-        )
-    });
-
-    krate
+    let lint_store = unerased_lint_store(tcx);
+    rustc_lint::check_ast_node(
+        sess,
+        false,
+        lint_store,
+        tcx.registered_tools(()),
+        Some(lint_buffer),
+        rustc_lint::BuiltinCombinedEarlyLintPass::new(),
+        (&**krate, &*krate.attrs),
+    )
 }
 
 // Returns all the paths that correspond to generated files.
@@ -401,12 +409,12 @@ where
 }
 
 fn output_contains_path(output_paths: &[PathBuf], input_path: &Path) -> bool {
-    let input_path = input_path.canonicalize().ok();
+    let input_path = try_canonicalize(input_path).ok();
     if input_path.is_none() {
         return false;
     }
     let check = |output_path: &PathBuf| {
-        if output_path.canonicalize().ok() == input_path { Some(()) } else { None }
+        if try_canonicalize(output_path).ok() == input_path { Some(()) } else { None }
     };
     check_output(output_paths, check).is_some()
 }
@@ -438,13 +446,9 @@ fn escape_dep_env(symbol: Symbol) -> String {
     escaped
 }
 
-fn write_out_deps(
-    sess: &Session,
-    cstore: &CrateStoreDyn,
-    outputs: &OutputFilenames,
-    out_filenames: &[PathBuf],
-) {
+fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[PathBuf]) {
     // Write out dependency rules to the dep-info file if requested
+    let sess = tcx.sess;
     if !sess.opts.output_types.contains_key(&OutputType::DepInfo) {
         return;
     }
@@ -492,9 +496,8 @@ fn write_out_deps(
                 }
             }
 
-            let cstore = cstore.as_any().downcast_ref::<CStore>().unwrap();
-            for cnum in cstore.crates_untracked() {
-                let source = cstore.crate_source_untracked(cnum);
+            for &cnum in tcx.crates(()) {
+                let source = tcx.used_crate_source(cnum);
                 if let Some((path, _)) = &source.dylib {
                     files.push(escape_dep_filename(&path.display().to_string()));
                 }
@@ -557,9 +560,10 @@ fn resolver_for_lowering<'tcx>(
     (): (),
 ) -> &'tcx Steal<(ty::ResolverAstLowering, Lrc<ast::Crate>)> {
     let arenas = Resolver::arenas();
-    let krate = tcx.crate_for_resolver(()).steal();
-    let mut resolver = Resolver::new(tcx, &krate, &arenas);
-    let krate = configure_and_expand(krate, &mut resolver);
+    let _ = tcx.registered_tools(()); // Uses `crate_for_resolver`.
+    let (krate, pre_configured_attrs) = tcx.crate_for_resolver(()).steal();
+    let mut resolver = Resolver::new(tcx, &pre_configured_attrs, krate.spans.inner_span, &arenas);
+    let krate = configure_and_expand(krate, &pre_configured_attrs, &mut resolver);
 
     // Make sure we don't mutate the cstore from here on.
     tcx.untracked().cstore.leak();
@@ -607,7 +611,7 @@ fn output_filenames(tcx: TyCtxt<'_>, (): ()) -> Arc<OutputFilenames> {
         }
     }
 
-    write_out_deps(sess, &*tcx.cstore_untracked(), &outputs, &output_paths);
+    write_out_deps(tcx, &outputs, &output_paths);
 
     let only_dep_info = sess.opts.output_types.contains_key(&OutputType::DepInfo)
         && sess.opts.output_types.len() == 1;
@@ -629,6 +633,7 @@ pub static DEFAULT_QUERY_PROVIDERS: LazyLock<Providers> = LazyLock::new(|| {
     providers.hir_crate = rustc_ast_lowering::lower_to_hir;
     providers.output_filenames = output_filenames;
     providers.resolver_for_lowering = resolver_for_lowering;
+    providers.early_lint_checks = early_lint_checks;
     proc_macro_decls::provide(providers);
     rustc_const_eval::provide(providers);
     rustc_middle::hir::provide(providers);
@@ -637,6 +642,7 @@ pub static DEFAULT_QUERY_PROVIDERS: LazyLock<Providers> = LazyLock::new(|| {
     rustc_mir_transform::provide(providers);
     rustc_monomorphize::provide(providers);
     rustc_privacy::provide(providers);
+    rustc_resolve::provide(providers);
     rustc_hir_analysis::provide(providers);
     rustc_hir_typeck::provide(providers);
     ty::provide(providers);
@@ -759,7 +765,7 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> {
         parallel!(
             {
                 sess.time("match_checking", || {
-                    tcx.hir().par_body_owners(|def_id| tcx.ensure().check_match(def_id.to_def_id()))
+                    tcx.hir().par_body_owners(|def_id| tcx.ensure().check_match(def_id))
                 });
             },
             {
diff --git a/compiler/rustc_interface/src/proc_macro_decls.rs b/compiler/rustc_interface/src/proc_macro_decls.rs
index 9bf7778bf..1c58caa03 100644
--- a/compiler/rustc_interface/src/proc_macro_decls.rs
+++ b/compiler/rustc_interface/src/proc_macro_decls.rs
@@ -1,3 +1,4 @@
+use rustc_ast::attr;
 use rustc_hir::def_id::LocalDefId;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::TyCtxt;
@@ -8,7 +9,7 @@ fn proc_macro_decls_static(tcx: TyCtxt<'_>, (): ()) -> Option<LocalDefId> {
 
     for id in tcx.hir().items() {
         let attrs = tcx.hir().attrs(id.hir_id());
-        if tcx.sess.contains_name(attrs, sym::rustc_proc_macro_decls) {
+        if attr::contains_name(attrs, sym::rustc_proc_macro_decls) {
             decls = Some(id.owner_id.def_id);
         }
     }
diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs
index a96cc95a3..818f450a5 100644
--- a/compiler/rustc_interface/src/queries.rs
+++ b/compiler/rustc_interface/src/queries.rs
@@ -7,7 +7,7 @@ use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::CodegenResults;
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::{AppendOnlyVec, Lrc, OnceCell, RwLock, WorkerLocal};
+use rustc_data_structures::sync::{AppendOnlyIndexVec, Lrc, OnceCell, RwLock, WorkerLocal};
 use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::definitions::Definitions;
 use rustc_incremental::DepGraphFuture;
@@ -88,8 +88,9 @@ pub struct Queries<'tcx> {
 
     dep_graph_future: Query<Option<DepGraphFuture>>,
     parse: Query<ast::Crate>,
+    pre_configure: Query<(ast::Crate, ast::AttrVec)>,
     crate_name: Query<Symbol>,
-    register_plugins: Query<(ast::Crate, Lrc<LintStore>)>,
+    register_plugins: Query<(ast::Crate, ast::AttrVec, Lrc<LintStore>)>,
     dep_graph: Query<DepGraph>,
     // This just points to what's in `gcx_cell`.
     gcx: Query<&'tcx GlobalCtxt<'tcx>>,
@@ -106,6 +107,7 @@ impl<'tcx> Queries<'tcx> {
             hir_arena: WorkerLocal::new(|_| rustc_hir::Arena::default()),
             dep_graph_future: Default::default(),
             parse: Default::default(),
+            pre_configure: Default::default(),
             crate_name: Default::default(),
             register_plugins: Default::default(),
             dep_graph: Default::default(),
@@ -133,17 +135,36 @@ impl<'tcx> Queries<'tcx> {
             .compute(|| passes::parse(self.session()).map_err(|mut parse_error| parse_error.emit()))
     }
 
-    pub fn register_plugins(&self) -> Result<QueryResult<'_, (ast::Crate, Lrc<LintStore>)>> {
+    pub fn pre_configure(&self) -> Result<QueryResult<'_, (ast::Crate, ast::AttrVec)>> {
+        self.pre_configure.compute(|| {
+            let mut krate = self.parse()?.steal();
+
+            let sess = self.session();
+            rustc_builtin_macros::cmdline_attrs::inject(
+                &mut krate,
+                &sess.parse_sess,
+                &sess.opts.unstable_opts.crate_attr,
+            );
+
+            let pre_configured_attrs =
+                rustc_expand::config::pre_configure_attrs(sess, &krate.attrs);
+            Ok((krate, pre_configured_attrs))
+        })
+    }
+
+    pub fn register_plugins(
+        &self,
+    ) -> Result<QueryResult<'_, (ast::Crate, ast::AttrVec, Lrc<LintStore>)>> {
         self.register_plugins.compute(|| {
             let crate_name = *self.crate_name()?.borrow();
-            let krate = self.parse()?.steal();
+            let (krate, pre_configured_attrs) = self.pre_configure()?.steal();
 
             let empty: &(dyn Fn(&Session, &mut LintStore) + Sync + Send) = &|_, _| {};
-            let (krate, lint_store) = passes::register_plugins(
+            let lint_store = passes::register_plugins(
                 self.session(),
                 &*self.codegen_backend().metadata_loader(),
                 self.compiler.register_lints.as_deref().unwrap_or_else(|| empty),
-                krate,
+                &pre_configured_attrs,
                 crate_name,
             )?;
 
@@ -154,17 +175,17 @@ impl<'tcx> Queries<'tcx> {
             // called, which happens within passes::register_plugins().
             self.dep_graph_future().ok();
 
-            Ok((krate, Lrc::new(lint_store)))
+            Ok((krate, pre_configured_attrs, Lrc::new(lint_store)))
         })
     }
 
     fn crate_name(&self) -> Result<QueryResult<'_, Symbol>> {
         self.crate_name.compute(|| {
             Ok({
-                let parse_result = self.parse()?;
-                let krate = parse_result.borrow();
+                let pre_configure_result = self.pre_configure()?;
+                let (_, pre_configured_attrs) = &*pre_configure_result.borrow();
                 // parse `#[crate_name]` even if `--crate-name` was passed, to make sure it matches.
-                find_crate_name(self.session(), &krate.attrs)
+                find_crate_name(self.session(), pre_configured_attrs)
             })
         })
     }
@@ -188,13 +209,13 @@ impl<'tcx> Queries<'tcx> {
     pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>> {
         self.gcx.compute(|| {
             let crate_name = *self.crate_name()?.borrow();
-            let (krate, lint_store) = self.register_plugins()?.steal();
+            let (krate, pre_configured_attrs, lint_store) = self.register_plugins()?.steal();
 
             let sess = self.session();
 
             let cstore = RwLock::new(Box::new(CStore::new(sess)) as _);
             let definitions = RwLock::new(Definitions::new(sess.local_stable_crate_id()));
-            let source_span = AppendOnlyVec::new();
+            let source_span = AppendOnlyIndexVec::new();
             let _id = source_span.push(krate.spans.inner_span);
             debug_assert_eq!(_id, CRATE_DEF_ID);
             let untracked = Untracked { cstore, source_span, definitions };
@@ -215,7 +236,7 @@ impl<'tcx> Queries<'tcx> {
                 feed.crate_name(crate_name);
 
                 let feed = tcx.feed_unit_query();
-                feed.crate_for_resolver(tcx.arena.alloc(Steal::new(krate)));
+                feed.crate_for_resolver(tcx.arena.alloc(Steal::new((krate, pre_configured_attrs))));
                 feed.metadata_loader(
                     tcx.arena.alloc(Steal::new(self.codegen_backend().metadata_loader())),
                 );
@@ -284,7 +305,11 @@ impl<'tcx> Queries<'tcx> {
         let codegen_backend = self.codegen_backend().clone();
 
         let (crate_hash, prepare_outputs, dep_graph) = self.global_ctxt()?.enter(|tcx| {
-            (tcx.crate_hash(LOCAL_CRATE), tcx.output_filenames(()).clone(), tcx.dep_graph.clone())
+            (
+                if tcx.sess.needs_crate_hash() { Some(tcx.crate_hash(LOCAL_CRATE)) } else { None },
+                tcx.output_filenames(()).clone(),
+                tcx.dep_graph.clone(),
+            )
         });
         let ongoing_codegen = self.ongoing_codegen()?.steal();
 
@@ -308,7 +333,8 @@ pub struct Linker {
     // compilation outputs
     dep_graph: DepGraph,
     prepare_outputs: Arc<OutputFilenames>,
-    crate_hash: Svh,
+    // Only present when incr. comp. is enabled.
+    crate_hash: Option<Svh>,
     ongoing_codegen: Box<dyn Any>,
 }
 
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 18d84a702..10dfd32d4 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -2,8 +2,10 @@
 use crate::interface::parse_cfgspecs;
 
 use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::profiling::TimePassesFormat;
 use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig};
 use rustc_session::config::rustc_optgroups;
+use rustc_session::config::DebugInfo;
 use rustc_session::config::Input;
 use rustc_session::config::InstrumentXRay;
 use rustc_session::config::TraitSolver;
@@ -573,7 +575,7 @@ fn test_codegen_options_tracking_hash() {
     tracked!(code_model, Some(CodeModel::Large));
     tracked!(control_flow_guard, CFGuard::Checks);
     tracked!(debug_assertions, Some(true));
-    tracked!(debuginfo, 0xdeadbeef);
+    tracked!(debuginfo, DebugInfo::Limited);
     tracked!(embed_bitcode, false);
     tracked!(force_frame_pointers, Some(false));
     tracked!(force_unwind_tables, Some(true));
@@ -699,6 +701,7 @@ fn test_unstable_options_tracking_hash() {
     untracked!(threads, 99);
     untracked!(time_llvm_passes, true);
     untracked!(time_passes, true);
+    untracked!(time_passes_format, TimePassesFormat::Json);
     untracked!(trace_macros, true);
     untracked!(track_diagnostics, true);
     untracked!(trim_diagnostic_paths, false);
@@ -744,6 +747,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(emit_thin_lto, false);
     tracked!(export_executable_symbols, true);
     tracked!(fewer_names, Some(true));
+    tracked!(flatten_format_args, true);
     tracked!(force_unstable_if_unmarked, true);
     tracked!(fuel, Some(("abc".to_string(), 99)));
     tracked!(function_sections, Some(false));
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index e5d2fb2ea..612903810 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -34,7 +34,7 @@ pub type MakeBackendFn = fn() -> Box<dyn CodegenBackend>;
 /// specific features (SSE, NEON etc.).
 ///
 /// This is performed by checking whether a set of permitted features
-/// is available on the target machine, by querying LLVM.
+/// is available on the target machine, by querying the codegen backend.
 pub fn add_configuration(
     cfg: &mut CrateConfig,
     sess: &mut Session,
@@ -110,7 +110,7 @@ pub fn create_session(
     add_configuration(&mut cfg, &mut sess, &*codegen_backend);
 
     let mut check_cfg = config::to_crate_check_config(check_cfg);
-    check_cfg.fill_well_known();
+    check_cfg.fill_well_known(&sess.target);
 
     sess.parse_sess.config = cfg;
     sess.parse_sess.check_config = check_cfg;
@@ -183,7 +183,7 @@ pub(crate) fn run_in_thread_pool_with_globals<F: FnOnce() -> R + Send, R: Send>(
                     .try_collect_active_jobs()
                     .expect("active jobs shouldn't be locked in deadlock handler")
             });
-            let registry = rustc_rayon_core::Registry::current();
+            let registry = rayon_core::Registry::current();
             thread::spawn(move || deadlock(query_map, &registry));
         });
     if let Some(size) = get_stack_size() {
@@ -505,7 +505,7 @@ pub fn build_output_filenames(attrs: &[ast::Attribute], sess: &Session) -> Outpu
                 .opts
                 .crate_name
                 .clone()
-                .or_else(|| rustc_attr::find_crate_name(sess, attrs).map(|n| n.to_string()))
+                .or_else(|| rustc_attr::find_crate_name(attrs).map(|n| n.to_string()))
                 .unwrap_or_else(|| sess.io.input.filestem().to_owned());
 
             OutputFilenames::new(
diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index 8507ca9d8..bb4d91247 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -298,10 +298,10 @@ where
         }
         let tail = &tail[first_non_space..];
         if let Some(c) = tail.chars().nth(0) {
-            // For error reporting, we would like the span to contain the character that was not
-            // skipped. The +1 is necessary to account for the leading \ that started the escape.
-            let end = start + first_non_space + c.len_utf8() + 1;
             if c.is_whitespace() {
+                // For error reporting, we would like the span to contain the character that was not
+                // skipped. The +1 is necessary to account for the leading \ that started the escape.
+                let end = start + first_non_space + c.len_utf8() + 1;
                 callback(start..end, Err(EscapeError::UnskippedWhitespaceWarning));
             }
         }
diff --git a/compiler/rustc_lint/locales/en-US.ftl b/compiler/rustc_lint/messages.ftl
index 68e62c978..db15b176d 100644
--- a/compiler/rustc_lint/locales/en-US.ftl
+++ b/compiler/rustc_lint/messages.ftl
@@ -91,7 +91,7 @@ lint_ty_qualified = usage of qualified `ty::{$ty}`
 lint_lintpass_by_hand = implementing `LintPass` by hand
     .help = try using `declare_lint_pass!` or `impl_lint_pass!` instead
 
-lint_non_existant_doc_keyword = found non-existing keyword `{$keyword}` used in `#[doc(keyword = \"...\")]`
+lint_non_existent_doc_keyword = found non-existing keyword `{$keyword}` used in `#[doc(keyword = "...")]`
     .help = only existing keywords are allowed in core/std
 
 lint_diag_out_of_impl =
@@ -107,7 +107,7 @@ lint_cstring_ptr = getting the inner pointer of a temporary `CString`
     .note = pointers do not have a lifetime; when calling `as_ptr` the `CString` will be deallocated at the end of the statement because nothing is referencing it as far as the type system is concerned
     .help = for more information, see https://doc.rust-lang.org/reference/destructors.html
 
-lint_multple_supertrait_upcastable = `{$ident}` is object-safe and has multiple supertraits
+lint_multiple_supertrait_upcastable = `{$ident}` is object-safe and has multiple supertraits
 
 lint_identifier_non_ascii_char = identifier contains non-ASCII characters
 
@@ -197,7 +197,9 @@ lint_drop_glue =
     types that do not implement `Drop` can still have drop glue, consider instead using `{$needs_drop}` to detect whether a type is trivially dropped
 
 lint_range_endpoint_out_of_range = range endpoint is out of range for `{$ty}`
-    .suggestion = use an inclusive range instead
+
+lint_range_use_inclusive_range = use an inclusive range instead
+
 
 lint_overflowing_bin_hex = literal out of range for `{$ty}`
     .negative_note = the literal `{$lit}` (decimal `{$dec}`) does not fit into the type `{$ty}`
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index 5b2100b5d..6b387df78 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -58,7 +58,6 @@ use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, LocalDefIdSet, CRATE_DEF_ID};
 use rustc_hir::intravisit::FnKind as HirFnKind;
 use rustc_hir::{Body, FnDecl, ForeignItemKind, GenericParamKind, Node, PatKind, PredicateOrigin};
-use rustc_index::vec::Idx;
 use rustc_middle::lint::in_external_macro;
 use rustc_middle::ty::layout::{LayoutError, LayoutOf};
 use rustc_middle::ty::print::with_no_trimmed_paths;
@@ -69,7 +68,7 @@ use rustc_span::edition::Edition;
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{BytePos, InnerSpan, Span};
-use rustc_target::abi::{Abi, VariantIdx};
+use rustc_target::abi::{Abi, FIRST_VARIANT};
 use rustc_trait_selection::infer::{InferCtxtExt, TyCtxtInferExt};
 use rustc_trait_selection::traits::{self, misc::type_allowed_to_implement_copy};
 
@@ -358,29 +357,29 @@ impl EarlyLintPass for UnsafeCode {
             }
 
             ast::ItemKind::Fn(..) => {
-                if let Some(attr) = cx.sess().find_by_name(&it.attrs, sym::no_mangle) {
+                if let Some(attr) = attr::find_by_name(&it.attrs, sym::no_mangle) {
                     self.report_unsafe(cx, attr.span, BuiltinUnsafe::NoMangleFn);
                 }
 
-                if let Some(attr) = cx.sess().find_by_name(&it.attrs, sym::export_name) {
+                if let Some(attr) = attr::find_by_name(&it.attrs, sym::export_name) {
                     self.report_unsafe(cx, attr.span, BuiltinUnsafe::ExportNameFn);
                 }
 
-                if let Some(attr) = cx.sess().find_by_name(&it.attrs, sym::link_section) {
+                if let Some(attr) = attr::find_by_name(&it.attrs, sym::link_section) {
                     self.report_unsafe(cx, attr.span, BuiltinUnsafe::LinkSectionFn);
                 }
             }
 
             ast::ItemKind::Static(..) => {
-                if let Some(attr) = cx.sess().find_by_name(&it.attrs, sym::no_mangle) {
+                if let Some(attr) = attr::find_by_name(&it.attrs, sym::no_mangle) {
                     self.report_unsafe(cx, attr.span, BuiltinUnsafe::NoMangleStatic);
                 }
 
-                if let Some(attr) = cx.sess().find_by_name(&it.attrs, sym::export_name) {
+                if let Some(attr) = attr::find_by_name(&it.attrs, sym::export_name) {
                     self.report_unsafe(cx, attr.span, BuiltinUnsafe::ExportNameStatic);
                 }
 
-                if let Some(attr) = cx.sess().find_by_name(&it.attrs, sym::link_section) {
+                if let Some(attr) = attr::find_by_name(&it.attrs, sym::link_section) {
                     self.report_unsafe(cx, attr.span, BuiltinUnsafe::LinkSectionStatic);
                 }
             }
@@ -391,10 +390,10 @@ impl EarlyLintPass for UnsafeCode {
 
     fn check_impl_item(&mut self, cx: &EarlyContext<'_>, it: &ast::AssocItem) {
         if let ast::AssocItemKind::Fn(..) = it.kind {
-            if let Some(attr) = cx.sess().find_by_name(&it.attrs, sym::no_mangle) {
+            if let Some(attr) = attr::find_by_name(&it.attrs, sym::no_mangle) {
                 self.report_unsafe(cx, attr.span, BuiltinUnsafe::NoMangleMethod);
             }
-            if let Some(attr) = cx.sess().find_by_name(&it.attrs, sym::export_name) {
+            if let Some(attr) = attr::find_by_name(&it.attrs, sym::export_name) {
                 self.report_unsafe(cx, attr.span, BuiltinUnsafe::ExportNameMethod);
             }
         }
@@ -1123,12 +1122,12 @@ impl<'tcx> LateLintPass<'tcx> for InvalidNoMangleItems {
         };
         match it.kind {
             hir::ItemKind::Fn(.., ref generics, _) => {
-                if let Some(no_mangle_attr) = cx.sess().find_by_name(attrs, sym::no_mangle) {
+                if let Some(no_mangle_attr) = attr::find_by_name(attrs, sym::no_mangle) {
                     check_no_mangle_on_generic_fn(no_mangle_attr, None, generics, it.span);
                 }
             }
             hir::ItemKind::Const(..) => {
-                if cx.sess().contains_name(attrs, sym::no_mangle) {
+                if attr::contains_name(attrs, sym::no_mangle) {
                     // account for "pub const" (#45562)
                     let start = cx
                         .tcx
@@ -1152,9 +1151,8 @@ impl<'tcx> LateLintPass<'tcx> for InvalidNoMangleItems {
             hir::ItemKind::Impl(hir::Impl { generics, items, .. }) => {
                 for it in *items {
                     if let hir::AssocItemKind::Fn { .. } = it.kind {
-                        if let Some(no_mangle_attr) = cx
-                            .sess()
-                            .find_by_name(cx.tcx.hir().attrs(it.id.hir_id()), sym::no_mangle)
+                        if let Some(no_mangle_attr) =
+                            attr::find_by_name(cx.tcx.hir().attrs(it.id.hir_id()), sym::no_mangle)
                         {
                             check_no_mangle_on_generic_fn(
                                 no_mangle_attr,
@@ -1306,7 +1304,7 @@ impl<'tcx> LateLintPass<'tcx> for UngatedAsyncFnTrackCaller {
         if fn_kind.asyncness() == IsAsync::Async
             && !cx.tcx.features().closure_track_caller
             // Now, check if the function has the `#[track_caller]` attribute
-            && let Some(attr) = cx.tcx.get_attr(def_id.to_def_id(), sym::track_caller)
+            && let Some(attr) = cx.tcx.get_attr(def_id, sym::track_caller)
         {
             cx.emit_spanned_lint(UNGATED_ASYNC_FN_TRACK_CALLER, attr.span, BuiltinUngatedAsyncFnTrackCaller {
                 label: span,
@@ -1601,7 +1599,7 @@ impl<'tcx> LateLintPass<'tcx> for TrivialConstraints {
                     // Ignore projections, as they can only be global
                     // if the trait bound is global
                     Clause(Clause::Projection(..)) |
-                    AliasEq(..) |
+                    AliasRelate(..) |
                     // Ignore bounds that a user can't type
                     WellFormed(..) |
                     ObjectSafe(..) |
@@ -1836,7 +1834,7 @@ impl<'tcx> LateLintPass<'tcx> for UnnameableTestItems {
         }
 
         let attrs = cx.tcx.hir().attrs(it.hir_id());
-        if let Some(attr) = cx.sess().find_by_name(attrs, sym::rustc_test_marker) {
+        if let Some(attr) = attr::find_by_name(attrs, sym::rustc_test_marker) {
             cx.emit_spanned_lint(UNNAMEABLE_TEST_ITEMS, attr.span, BuiltinUnnameableTestItems);
         }
     }
@@ -1949,7 +1947,7 @@ impl KeywordIdents {
         };
 
         // Don't lint `r#foo`.
-        if cx.sess().parse_sess.raw_identifier_spans.borrow().contains(&ident.span) {
+        if cx.sess().parse_sess.raw_identifier_spans.contains(ident.span) {
             return;
         }
 
@@ -2748,10 +2746,7 @@ impl ClashingExternDeclarations {
                 // information, we could have codegen_fn_attrs also give span information back for
                 // where the attribute was defined. However, until this is found to be a
                 // bottleneck, this does just fine.
-                (
-                    overridden_link_name,
-                    tcx.get_attr(fi.owner_id.to_def_id(), sym::link_name).unwrap().span,
-                )
+                (overridden_link_name, tcx.get_attr(fi.owner_id, sym::link_name).unwrap().span)
             })
         {
             SymbolName::Link(overridden_link_name, overridden_link_name_span)
@@ -2781,8 +2776,7 @@ impl ClashingExternDeclarations {
 
             // Given a transparent newtype, reach through and grab the inner
             // type unless the newtype makes the type non-null.
-            let non_transparent_ty = |ty: Ty<'tcx>| -> Ty<'tcx> {
-                let mut ty = ty;
+            let non_transparent_ty = |mut ty: Ty<'tcx>| -> Ty<'tcx> {
                 loop {
                     if let ty::Adt(def, substs) = *ty.kind() {
                         let is_transparent = def.repr().transparent();
@@ -2792,14 +2786,14 @@ impl ClashingExternDeclarations {
                             ty, is_transparent, is_non_null
                         );
                         if is_transparent && !is_non_null {
-                            debug_assert!(def.variants().len() == 1);
-                            let v = &def.variant(VariantIdx::new(0));
-                            ty = transparent_newtype_field(tcx, v)
-                                .expect(
-                                    "single-variant transparent structure with zero-sized field",
-                                )
-                                .ty(tcx, substs);
-                            continue;
+                            debug_assert_eq!(def.variants().len(), 1);
+                            let v = &def.variant(FIRST_VARIANT);
+                            // continue with `ty`'s non-ZST field,
+                            // otherwise `ty` is a ZST and we can return
+                            if let Some(field) = transparent_newtype_field(tcx, v) {
+                                ty = field.ty(tcx, substs);
+                                continue;
+                            }
                         }
                     }
                     debug!("non_transparent_ty -> {:?}", ty);
@@ -2813,10 +2807,8 @@ impl ClashingExternDeclarations {
             if !seen_types.insert((a, b)) {
                 // We've encountered a cycle. There's no point going any further -- the types are
                 // structurally the same.
-                return true;
-            }
-            let tcx = cx.tcx;
-            if a == b {
+                true
+            } else if a == b {
                 // All nominally-same types are structurally same, too.
                 true
             } else {
diff --git a/compiler/rustc_lint/src/context.rs b/compiler/rustc_lint/src/context.rs
index f5a711315..626c09fea 100644
--- a/compiler/rustc_lint/src/context.rs
+++ b/compiler/rustc_lint/src/context.rs
@@ -910,6 +910,10 @@ pub trait LintContext: Sized {
                         Applicability::MachineApplicable,
                     );
                 }
+                BuiltinLintDiagnostics::AmbiguousGlobReexports { name, namespace, first_reexport_span, duplicate_reexport_span } => {
+                    db.span_label(first_reexport_span, format!("the name `{}` in the {} namespace is first re-exported here", name, namespace));
+                    db.span_label(duplicate_reexport_span, format!("but the name `{}` in the {} namespace is also re-exported here", name, namespace));
+                }
             }
             // Rewrap `db`, and pass control to the user.
             decorate(db)
diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs
index 337a19dd0..65607d718 100644
--- a/compiler/rustc_lint/src/early.rs
+++ b/compiler/rustc_lint/src/early.rs
@@ -224,8 +224,7 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
             ast::ExprKind::Closure(box ast::Closure {
                 asyncness: ast::Async::Yes { closure_id, .. },
                 ..
-            })
-            | ast::ExprKind::Async(_, closure_id, ..) => self.check_id(closure_id),
+            }) => self.check_id(closure_id),
             _ => {}
         }
     }
@@ -341,7 +340,7 @@ pub trait EarlyCheckNode<'a>: Copy {
         'a: 'b;
 }
 
-impl<'a> EarlyCheckNode<'a> for &'a ast::Crate {
+impl<'a> EarlyCheckNode<'a> for (&'a ast::Crate, &'a [ast::Attribute]) {
     fn id(self) -> ast::NodeId {
         ast::CRATE_NODE_ID
     }
@@ -349,15 +348,15 @@ impl<'a> EarlyCheckNode<'a> for &'a ast::Crate {
     where
         'a: 'b,
     {
-        &self.attrs
+        &self.1
     }
     fn check<'b, T: EarlyLintPass>(self, cx: &mut EarlyContextAndPass<'b, T>)
     where
         'a: 'b,
     {
-        lint_callback!(cx, check_crate, self);
-        ast_visit::walk_crate(cx, self);
-        lint_callback!(cx, check_crate_post, self);
+        lint_callback!(cx, check_crate, self.0);
+        ast_visit::walk_crate(cx, self.0);
+        lint_callback!(cx, check_crate_post, self.0);
     }
 }
 
diff --git a/compiler/rustc_lint/src/for_loops_over_fallibles.rs b/compiler/rustc_lint/src/for_loops_over_fallibles.rs
index a3367ae4a..7b58bf03b 100644
--- a/compiler/rustc_lint/src/for_loops_over_fallibles.rs
+++ b/compiler/rustc_lint/src/for_loops_over_fallibles.rs
@@ -11,6 +11,7 @@ use rustc_hir as hir;
 use rustc_infer::{infer::TyCtxtInferExt, traits::ObligationCause};
 use rustc_middle::ty::{self, List};
 use rustc_span::{sym, Span};
+use rustc_trait_selection::traits::ObligationCtxt;
 
 declare_lint! {
     /// The `for_loops_over_fallibles` lint checks for `for` loops over `Option` or `Result` values.
@@ -136,20 +137,22 @@ fn suggest_question_mark<'tcx>(
 
     let ty = substs.type_at(0);
     let infcx = cx.tcx.infer_ctxt().build();
+    let ocx = ObligationCtxt::new(&infcx);
+
     let body_def_id = cx.tcx.hir().body_owner_def_id(body_id);
     let cause = ObligationCause::new(
         span,
         body_def_id,
         rustc_infer::traits::ObligationCauseCode::MiscObligation,
     );
-    let errors = rustc_trait_selection::traits::fully_solve_bound(
-        &infcx,
+
+    ocx.register_bound(
         cause,
-        ty::ParamEnv::empty(),
+        cx.param_env,
         // Erase any region vids from the type, which may not be resolved
         infcx.tcx.erase_regions(ty),
         into_iterator_did,
     );
 
-    errors.is_empty()
+    ocx.select_all_or_error().is_empty()
 }
diff --git a/compiler/rustc_lint/src/internal.rs b/compiler/rustc_lint/src/internal.rs
index 2fd0ef3cd..4ac589c2e 100644
--- a/compiler/rustc_lint/src/internal.rs
+++ b/compiler/rustc_lint/src/internal.rs
@@ -2,7 +2,7 @@
 //! Clippy.
 
 use crate::lints::{
-    BadOptAccessDiag, DefaultHashTypesDiag, DiagOutOfImpl, LintPassByHand, NonExistantDocKeyword,
+    BadOptAccessDiag, DefaultHashTypesDiag, DiagOutOfImpl, LintPassByHand, NonExistentDocKeyword,
     QueryInstability, TyQualified, TykindDiag, TykindKind, UntranslatableDiag,
 };
 use crate::{EarlyContext, EarlyLintPass, LateContext, LateLintPass, LintContext};
@@ -17,6 +17,11 @@ use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
 
 declare_tool_lint! {
+    /// The `default_hash_type` lint detects use of [`std::collections::HashMap`]/[`std::collections::HashSet`],
+    /// suggesting the use of `FxHashMap`/`FxHashSet`.
+    ///
+    /// This can help as `FxHasher` can perform better than the default hasher. DOS protection is not
+    /// required as input is assumed to be trusted.
     pub rustc::DEFAULT_HASH_TYPES,
     Allow,
     "forbid HashMap and HashSet and suggest the FxHash* variants",
@@ -67,6 +72,12 @@ fn typeck_results_of_method_fn<'tcx>(
 }
 
 declare_tool_lint! {
+    /// The `potential_query_instability` lint detects use of methods which can lead to
+    /// potential query instability, such as iterating over a `HashMap`.
+    ///
+    /// Due to the [incremental compilation](https://rustc-dev-guide.rust-lang.org/queries/incremental-compilation.html) model,
+    /// queries must return deterministic, stable results. `HashMap` iteration order can change between compilations,
+    /// and will introduce instability if query results expose the order.
     pub rustc::POTENTIAL_QUERY_INSTABILITY,
     Allow,
     "require explicit opt-in when using potentially unstable methods or functions",
@@ -92,6 +103,8 @@ impl LateLintPass<'_> for QueryStability {
 }
 
 declare_tool_lint! {
+    /// The `usage_of_ty_tykind` lint detects usages of `ty::TyKind::<kind>`,
+    /// where `ty::<kind>` would suffice.
     pub rustc::USAGE_OF_TY_TYKIND,
     Allow,
     "usage of `ty::TyKind` outside of the `ty::sty` module",
@@ -99,6 +112,8 @@ declare_tool_lint! {
 }
 
 declare_tool_lint! {
+    /// The `usage_of_qualified_ty` lint detects usages of `ty::TyKind`,
+    /// where `Ty` should be used instead.
     pub rustc::USAGE_OF_QUALIFIED_TY,
     Allow,
     "using `ty::{Ty,TyCtxt}` instead of importing it",
@@ -254,6 +269,8 @@ fn gen_args(segment: &PathSegment<'_>) -> String {
 }
 
 declare_tool_lint! {
+    /// The `lint_pass_impl_without_macro` detects manual implementations of a lint
+    /// pass, without using [`declare_lint_pass`] or [`impl_lint_pass`].
     pub rustc::LINT_PASS_IMPL_WITHOUT_MACRO,
     Allow,
     "`impl LintPass` without the `declare_lint_pass!` or `impl_lint_pass!` macros"
@@ -285,6 +302,8 @@ impl EarlyLintPass for LintPassImpl {
 }
 
 declare_tool_lint! {
+    /// The `existing_doc_keyword` lint detects use `#[doc()]` keywords
+    /// that don't exist, e.g. `#[doc(keyword = "..")]`.
     pub rustc::EXISTING_DOC_KEYWORD,
     Allow,
     "Check that documented keywords in std and core actually exist",
@@ -315,7 +334,7 @@ impl<'tcx> LateLintPass<'tcx> for ExistingDocKeyword {
                         cx.emit_spanned_lint(
                             EXISTING_DOC_KEYWORD,
                             attr.span,
-                            NonExistantDocKeyword { keyword },
+                            NonExistentDocKeyword { keyword },
                         );
                     }
                 }
@@ -325,6 +344,10 @@ impl<'tcx> LateLintPass<'tcx> for ExistingDocKeyword {
 }
 
 declare_tool_lint! {
+    /// The `untranslatable_diagnostic` lint detects diagnostics created
+    /// without using translatable Fluent strings.
+    ///
+    /// More details on translatable diagnostics can be found [here](https://rustc-dev-guide.rust-lang.org/diagnostics/translation.html).
     pub rustc::UNTRANSLATABLE_DIAGNOSTIC,
     Allow,
     "prevent creation of diagnostics which cannot be translated",
@@ -332,6 +355,11 @@ declare_tool_lint! {
 }
 
 declare_tool_lint! {
+    /// The `diagnostic_outside_of_impl` lint detects diagnostics created manually,
+    /// and inside an `IntoDiagnostic`/`AddToDiagnostic` implementation,
+    /// or a `#[derive(Diagnostic)]`/`#[derive(Subdiagnostic)]` expansion.
+    ///
+    /// More details on diagnostics implementations can be found [here](https://rustc-dev-guide.rust-lang.org/diagnostics/diagnostic-structs.html).
     pub rustc::DIAGNOSTIC_OUTSIDE_OF_IMPL,
     Allow,
     "prevent creation of diagnostics outside of `IntoDiagnostic`/`AddToDiagnostic` impls",
@@ -358,7 +386,7 @@ impl LateLintPass<'_> for Diagnostics {
         for (hir_id, parent) in cx.tcx.hir().parent_iter(expr.hir_id) {
             if let Some(owner_did) = hir_id.as_owner() {
                 found_parent_with_attr = found_parent_with_attr
-                    || cx.tcx.has_attr(owner_did.to_def_id(), sym::rustc_lint_diagnostics);
+                    || cx.tcx.has_attr(owner_did, sym::rustc_lint_diagnostics);
             }
 
             debug!(?parent);
@@ -396,6 +424,8 @@ impl LateLintPass<'_> for Diagnostics {
 }
 
 declare_tool_lint! {
+    /// The `bad_opt_access` lint detects accessing options by field instead of
+    /// the wrapper function.
     pub rustc::BAD_OPT_ACCESS,
     Deny,
     "prevent using options by field access when there is a wrapper function",
diff --git a/compiler/rustc_lint/src/let_underscore.rs b/compiler/rustc_lint/src/let_underscore.rs
index b83a9665f..3eefd1b0e 100644
--- a/compiler/rustc_lint/src/let_underscore.rs
+++ b/compiler/rustc_lint/src/let_underscore.rs
@@ -25,7 +25,7 @@ declare_lint! {
     ///
     /// fn main() {
     ///    #[warn(let_underscore_drop)]
-    ///     // SomeStuct is dropped immediately instead of at end of scope,
+    ///     // SomeStruct is dropped immediately instead of at end of scope,
     ///     // so "Dropping SomeStruct" is printed before "end of main".
     ///     // The order of prints would be reversed if SomeStruct was bound to
     ///     // a name (such as "_foo").
diff --git a/compiler/rustc_lint/src/levels.rs b/compiler/rustc_lint/src/levels.rs
index bc7488fab..bb863f095 100644
--- a/compiler/rustc_lint/src/levels.rs
+++ b/compiler/rustc_lint/src/levels.rs
@@ -3,7 +3,7 @@ use crate::{
     fluent_generated as fluent,
     late::unerased_lint_store,
     lints::{
-        DeprecatedLintName, IgnoredUnlessCrateSpecified, OverruledAtributeLint,
+        DeprecatedLintName, IgnoredUnlessCrateSpecified, OverruledAttributeLint,
         RenamedOrRemovedLint, RenamedOrRemovedLintSuggestion, UnknownLint, UnknownLintSuggestion,
     },
 };
@@ -128,7 +128,7 @@ fn lint_expectations(tcx: TyCtxt<'_>, (): ()) -> Vec<(LintExpectationId, LintExp
         },
         warn_about_weird_lints: false,
         store,
-        registered_tools: &tcx.resolutions(()).registered_tools,
+        registered_tools: &tcx.registered_tools(()),
     };
 
     builder.add_command_line();
@@ -156,7 +156,7 @@ fn shallow_lint_levels_on(tcx: TyCtxt<'_>, owner: hir::OwnerId) -> ShallowLintLe
         },
         warn_about_weird_lints: false,
         store,
-        registered_tools: &tcx.resolutions(()).registered_tools,
+        registered_tools: &tcx.registered_tools(()),
     };
 
     if owner == hir::CRATE_OWNER_ID {
@@ -266,12 +266,12 @@ impl LintLevelsProvider for QueryMapExpectationsWrapper<'_> {
         let LintExpectationId::Stable { attr_id: Some(attr_id), hir_id, attr_index, .. } = id else { bug!("unstable expectation id should already be mapped") };
         let key = LintExpectationId::Unstable { attr_id, lint_index: None };
 
-        if !self.unstable_to_stable_ids.contains_key(&key) {
-            self.unstable_to_stable_ids.insert(
-                key,
-                LintExpectationId::Stable { hir_id, attr_index, lint_index: None, attr_id: None },
-            );
-        }
+        self.unstable_to_stable_ids.entry(key).or_insert(LintExpectationId::Stable {
+            hir_id,
+            attr_index,
+            lint_index: None,
+            attr_id: None,
+        });
 
         self.expectations.push((id.normalize(), expectation));
     }
@@ -612,7 +612,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
                     self.emit_spanned_lint(
                         FORBIDDEN_LINT_GROUPS,
                         src.span().into(),
-                        OverruledAtributeLint {
+                        OverruledAttributeLint {
                             overruled: src.span(),
                             lint_level: level.as_str(),
                             lint_source: src.name(),
diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs
index 35dc533e5..b35785405 100644
--- a/compiler/rustc_lint/src/lib.rs
+++ b/compiler/rustc_lint/src/lib.rs
@@ -126,7 +126,7 @@ pub use rustc_session::lint::Level::{self, *};
 pub use rustc_session::lint::{BufferedEarlyLint, FutureIncompatibleInfo, Lint, LintId};
 pub use rustc_session::lint::{LintArray, LintPass};
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     levels::provide(providers);
diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs
index 20ab0af58..1d5e02369 100644
--- a/compiler/rustc_lint/src/lints.rs
+++ b/compiler/rustc_lint/src/lints.rs
@@ -806,9 +806,9 @@ pub struct TyQualified {
 pub struct LintPassByHand;
 
 #[derive(LintDiagnostic)]
-#[diag(lint_non_existant_doc_keyword)]
+#[diag(lint_non_existent_doc_keyword)]
 #[help]
-pub struct NonExistantDocKeyword {
+pub struct NonExistentDocKeyword {
     pub keyword: Symbol,
 }
 
@@ -875,7 +875,7 @@ impl AddToDiagnostic for NonBindingLetSub {
 // levels.rs
 #[derive(LintDiagnostic)]
 #[diag(lint_overruled_attribute)]
-pub struct OverruledAtributeLint<'a> {
+pub struct OverruledAttributeLint<'a> {
     #[label]
     pub overruled: Span,
     pub lint_level: &'a str,
@@ -947,7 +947,7 @@ pub struct CStringPtr {
 
 // multiple_supertrait_upcastable.rs
 #[derive(LintDiagnostic)]
-#[diag(lint_multple_supertrait_upcastable)]
+#[diag(lint_multiple_supertrait_upcastable)]
 pub struct MultipleSupertraitUpcastable {
     pub ident: Ident,
 }
@@ -1210,11 +1210,33 @@ impl<'a> DecorateLint<'a, ()> for DropGlue<'_> {
 #[diag(lint_range_endpoint_out_of_range)]
 pub struct RangeEndpointOutOfRange<'a> {
     pub ty: &'a str,
-    #[suggestion(code = "{start}..={literal}{suffix}", applicability = "machine-applicable")]
-    pub suggestion: Span,
-    pub start: String,
-    pub literal: u128,
-    pub suffix: &'a str,
+    #[subdiagnostic]
+    pub sub: UseInclusiveRange<'a>,
+}
+
+#[derive(Subdiagnostic)]
+pub enum UseInclusiveRange<'a> {
+    #[suggestion(
+        lint_range_use_inclusive_range,
+        code = "{start}..={literal}{suffix}",
+        applicability = "machine-applicable"
+    )]
+    WithoutParen {
+        #[primary_span]
+        sugg: Span,
+        start: String,
+        literal: u128,
+        suffix: &'a str,
+    },
+    #[multipart_suggestion(lint_range_use_inclusive_range, applicability = "machine-applicable")]
+    WithParen {
+        #[suggestion_part(code = "=")]
+        eq_sugg: Span,
+        #[suggestion_part(code = "{literal}{suffix}")]
+        lit_sugg: Span,
+        literal: u128,
+        suffix: &'a str,
+    },
 }
 
 #[derive(LintDiagnostic)]
@@ -1390,7 +1412,7 @@ pub struct UnusedOp<'a> {
     pub op: &'a str,
     #[label]
     pub label: Span,
-    #[suggestion(style = "verbose", code = "let _ = ", applicability = "machine-applicable")]
+    #[suggestion(style = "verbose", code = "let _ = ", applicability = "maybe-incorrect")]
     pub suggestion: Span,
 }
 
@@ -1400,7 +1422,7 @@ pub struct UnusedResult<'a> {
     pub ty: Ty<'a>,
 }
 
-// FIXME(davidtwco): this isn't properly translatable becauses of the
+// FIXME(davidtwco): this isn't properly translatable because of the
 // pre/post strings
 #[derive(LintDiagnostic)]
 #[diag(lint_unused_closure)]
@@ -1411,7 +1433,7 @@ pub struct UnusedClosure<'a> {
     pub post: &'a str,
 }
 
-// FIXME(davidtwco): this isn't properly translatable becauses of the
+// FIXME(davidtwco): this isn't properly translatable because of the
 // pre/post strings
 #[derive(LintDiagnostic)]
 #[diag(lint_unused_generator)]
@@ -1422,7 +1444,7 @@ pub struct UnusedGenerator<'a> {
     pub post: &'a str,
 }
 
-// FIXME(davidtwco): this isn't properly translatable becauses of the pre/post
+// FIXME(davidtwco): this isn't properly translatable because of the pre/post
 // strings
 pub struct UnusedDef<'a, 'b> {
     pub pre: &'a str,
@@ -1434,17 +1456,15 @@ pub struct UnusedDef<'a, 'b> {
 }
 
 #[derive(Subdiagnostic)]
-pub enum UnusedDefSuggestion {
-    #[suggestion(
-        lint_suggestion,
-        style = "verbose",
-        code = "let _ = ",
-        applicability = "machine-applicable"
-    )]
-    Default {
-        #[primary_span]
-        span: Span,
-    },
+#[suggestion(
+    lint_suggestion,
+    style = "verbose",
+    code = "let _ = ",
+    applicability = "maybe-incorrect"
+)]
+pub struct UnusedDefSuggestion {
+    #[primary_span]
+    pub span: Span,
 }
 
 // Needed because of def_path_str
diff --git a/compiler/rustc_lint/src/map_unit_fn.rs b/compiler/rustc_lint/src/map_unit_fn.rs
index 62e8b4fe9..7c692fee3 100644
--- a/compiler/rustc_lint/src/map_unit_fn.rs
+++ b/compiler/rustc_lint/src/map_unit_fn.rs
@@ -56,6 +56,7 @@ impl<'tcx> LateLintPass<'tcx> for MapUnitFn {
                         return;
                     }
                     let arg_ty = cx.typeck_results().expr_ty(&args[0]);
+                    let default_span = args[0].span;
                     if let ty::FnDef(id, _) = arg_ty.kind() {
                         let fn_ty = cx.tcx.fn_sig(id).skip_binder();
                         let ret_ty = fn_ty.output().skip_binder();
@@ -64,7 +65,10 @@ impl<'tcx> LateLintPass<'tcx> for MapUnitFn {
                                 MAP_UNIT_FN,
                                 span,
                                 MappingToUnit {
-                                    function_label: cx.tcx.span_of_impl(*id).unwrap(),
+                                    function_label: cx
+                                        .tcx
+                                        .span_of_impl(*id)
+                                        .unwrap_or(default_span),
                                     argument_label: args[0].span,
                                     map_label: arg_ty.default_span(cx.tcx),
                                     suggestion: path.ident.span,
@@ -80,7 +84,10 @@ impl<'tcx> LateLintPass<'tcx> for MapUnitFn {
                                 MAP_UNIT_FN,
                                 span,
                                 MappingToUnit {
-                                    function_label: cx.tcx.span_of_impl(*id).unwrap(),
+                                    function_label: cx
+                                        .tcx
+                                        .span_of_impl(*id)
+                                        .unwrap_or(default_span),
                                     argument_label: args[0].span,
                                     map_label: arg_ty.default_span(cx.tcx),
                                     suggestion: path.ident.span,
diff --git a/compiler/rustc_lint/src/methods.rs b/compiler/rustc_lint/src/methods.rs
index 3045fc1a4..4c25d94a1 100644
--- a/compiler/rustc_lint/src/methods.rs
+++ b/compiler/rustc_lint/src/methods.rs
@@ -2,9 +2,9 @@ use crate::lints::CStringPtr;
 use crate::LateContext;
 use crate::LateLintPass;
 use crate::LintContext;
-use rustc_hir::{Expr, ExprKind, PathSegment};
+use rustc_hir::{Expr, ExprKind};
 use rustc_middle::ty;
-use rustc_span::{symbol::sym, ExpnKind, Span};
+use rustc_span::{symbol::sym, Span};
 
 declare_lint! {
     /// The `temporary_cstring_as_ptr` lint detects getting the inner pointer of
@@ -34,47 +34,14 @@ declare_lint! {
 
 declare_lint_pass!(TemporaryCStringAsPtr => [TEMPORARY_CSTRING_AS_PTR]);
 
-fn in_macro(span: Span) -> bool {
-    if span.from_expansion() {
-        !matches!(span.ctxt().outer_expn_data().kind, ExpnKind::Desugaring(..))
-    } else {
-        false
-    }
-}
-
-fn first_method_call<'tcx>(
-    expr: &'tcx Expr<'tcx>,
-) -> Option<(&'tcx PathSegment<'tcx>, &'tcx Expr<'tcx>)> {
-    if let ExprKind::MethodCall(path, receiver, args, ..) = &expr.kind {
-        if args.iter().any(|e| e.span.from_expansion()) || receiver.span.from_expansion() {
-            None
-        } else {
-            Some((path, *receiver))
-        }
-    } else {
-        None
-    }
-}
-
 impl<'tcx> LateLintPass<'tcx> for TemporaryCStringAsPtr {
     fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>) {
-        if in_macro(expr.span) {
-            return;
-        }
-
-        match first_method_call(expr) {
-            Some((path, unwrap_arg)) if path.ident.name == sym::as_ptr => {
-                let as_ptr_span = path.ident.span;
-                match first_method_call(unwrap_arg) {
-                    Some((path, receiver))
-                        if path.ident.name == sym::unwrap || path.ident.name == sym::expect =>
-                    {
-                        lint_cstring_as_ptr(cx, as_ptr_span, receiver, unwrap_arg);
-                    }
-                    _ => return,
-                }
-            }
-            _ => return,
+        if let ExprKind::MethodCall(as_ptr_path, as_ptr_receiver, ..) = expr.kind
+            && as_ptr_path.ident.name == sym::as_ptr
+            && let ExprKind::MethodCall(unwrap_path, unwrap_receiver, ..) = as_ptr_receiver.kind
+            && (unwrap_path.ident.name == sym::unwrap || unwrap_path.ident.name == sym::expect)
+        {
+            lint_cstring_as_ptr(cx, as_ptr_path.ident.span, unwrap_receiver, as_ptr_receiver);
         }
     }
 }
diff --git a/compiler/rustc_lint/src/non_ascii_idents.rs b/compiler/rustc_lint/src/non_ascii_idents.rs
index f130a9818..4af879b4e 100644
--- a/compiler/rustc_lint/src/non_ascii_idents.rs
+++ b/compiler/rustc_lint/src/non_ascii_idents.rs
@@ -250,7 +250,7 @@ impl EarlyLintPass for NonAsciiIdents {
             let latin_augmented_script_set = AugmentedScriptSet::for_char('A');
             script_states.insert(latin_augmented_script_set, ScriptSetUsage::Verified);
 
-            let mut has_suspicous = false;
+            let mut has_suspicious = false;
             for (symbol, &sp) in symbols.iter() {
                 let symbol_str = symbol.as_str();
                 for ch in symbol_str.chars() {
@@ -278,14 +278,14 @@ impl EarlyLintPass for NonAsciiIdents {
                             if !is_potential_mixed_script_confusable_char(ch) {
                                 ScriptSetUsage::Verified
                             } else {
-                                has_suspicous = true;
+                                has_suspicious = true;
                                 ScriptSetUsage::Suspicious(vec![ch], sp)
                             }
                         });
                 }
             }
 
-            if has_suspicous {
+            if has_suspicious {
                 let verified_augmented_script_sets = script_states
                     .iter()
                     .flat_map(|(k, v)| match v {
diff --git a/compiler/rustc_lint/src/nonstandard_style.rs b/compiler/rustc_lint/src/nonstandard_style.rs
index 71e2e66bd..9efc14849 100644
--- a/compiler/rustc_lint/src/nonstandard_style.rs
+++ b/compiler/rustc_lint/src/nonstandard_style.rs
@@ -328,8 +328,7 @@ impl<'tcx> LateLintPass<'tcx> for NonSnakeCase {
         let crate_ident = if let Some(name) = &cx.tcx.sess.opts.crate_name {
             Some(Ident::from_str(name))
         } else {
-            cx.sess()
-                .find_by_name(&cx.tcx.hir().attrs(hir::CRATE_HIR_ID), sym::crate_name)
+            attr::find_by_name(&cx.tcx.hir().attrs(hir::CRATE_HIR_ID), sym::crate_name)
                 .and_then(|attr| attr.meta())
                 .and_then(|meta| {
                     meta.name_value_literal().and_then(|lit| {
@@ -384,9 +383,7 @@ impl<'tcx> LateLintPass<'tcx> for NonSnakeCase {
         match &fk {
             FnKind::Method(ident, sig, ..) => match method_context(cx, id) {
                 MethodLateContext::PlainImpl => {
-                    if sig.header.abi != Abi::Rust
-                        && cx.tcx.has_attr(id.to_def_id(), sym::no_mangle)
-                    {
+                    if sig.header.abi != Abi::Rust && cx.tcx.has_attr(id, sym::no_mangle) {
                         return;
                     }
                     self.check_snake_case(cx, "method", ident);
@@ -398,7 +395,7 @@ impl<'tcx> LateLintPass<'tcx> for NonSnakeCase {
             },
             FnKind::ItemFn(ident, _, header) => {
                 // Skip foreign-ABI #[no_mangle] functions (Issue #31924)
-                if header.abi != Abi::Rust && cx.tcx.has_attr(id.to_def_id(), sym::no_mangle) {
+                if header.abi != Abi::Rust && cx.tcx.has_attr(id, sym::no_mangle) {
                     return;
                 }
                 self.check_snake_case(cx, "function", ident);
@@ -491,7 +488,7 @@ impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
     fn check_item(&mut self, cx: &LateContext<'_>, it: &hir::Item<'_>) {
         let attrs = cx.tcx.hir().attrs(it.hir_id());
         match it.kind {
-            hir::ItemKind::Static(..) if !cx.sess().contains_name(attrs, sym::no_mangle) => {
+            hir::ItemKind::Static(..) if !attr::contains_name(attrs, sym::no_mangle) => {
                 NonUpperCaseGlobals::check_upper_case(cx, "static variable", &it.ident);
             }
             hir::ItemKind::Const(..) => {
diff --git a/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs b/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
index 883a56cb3..f9d43fe22 100644
--- a/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
+++ b/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
@@ -27,6 +27,8 @@ declare_lint! {
     /// ### Example
     ///
     /// ```rust
+    /// #![feature(type_alias_impl_trait)]
+    ///
     /// trait Duh {}
     ///
     /// impl Duh for i32 {}
@@ -41,7 +43,9 @@ declare_lint! {
     ///     type Assoc = F;
     /// }
     ///
-    /// fn test() -> impl Trait<Assoc = impl Sized> {
+    /// type Tait = impl Sized;
+    ///
+    /// fn test() -> impl Trait<Assoc = Tait> {
     ///     42
     /// }
     /// ```
@@ -54,7 +58,7 @@ declare_lint! {
     ///
     /// Although the hidden type, `i32` does satisfy this bound, we do not
     /// consider the return type to be well-formed with this lint. It can be
-    /// fixed by changing `impl Sized` into `impl Sized + Send`.
+    /// fixed by changing `Tait = impl Sized` into `Tait = impl Sized + Send`.
     pub OPAQUE_HIDDEN_INFERRED_BOUND,
     Warn,
     "detects the use of nested `impl Trait` types in associated type bounds that are not general enough"
@@ -64,7 +68,7 @@ declare_lint_pass!(OpaqueHiddenInferredBound => [OPAQUE_HIDDEN_INFERRED_BOUND]);
 
 impl<'tcx> LateLintPass<'tcx> for OpaqueHiddenInferredBound {
     fn check_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx hir::Item<'tcx>) {
-        let hir::ItemKind::OpaqueTy(_) = &item.kind else { return; };
+        let hir::ItemKind::OpaqueTy(opaque) = &item.kind else { return; };
         let def_id = item.owner_id.def_id.to_def_id();
         let infcx = &cx.tcx.infer_ctxt().build();
         // For every projection predicate in the opaque type's explicit bounds,
@@ -81,6 +85,17 @@ impl<'tcx> LateLintPass<'tcx> for OpaqueHiddenInferredBound {
             // have opaques in them anyways.
             let Some(proj_term) = proj.term.ty() else { continue };
 
+            // HACK: `impl Trait<Assoc = impl Trait2>` from an RPIT is "ok"...
+            if let ty::Alias(ty::Opaque, opaque_ty) = *proj_term.kind()
+                && cx.tcx.parent(opaque_ty.def_id) == def_id
+                && matches!(
+                    opaque.origin,
+                    hir::OpaqueTyOrigin::FnReturn(_) | hir::OpaqueTyOrigin::AsyncFn(_)
+                )
+            {
+                continue;
+            }
+
             let proj_ty =
                 cx.tcx.mk_projection(proj.projection_ty.def_id, proj.projection_ty.substs);
             // For every instance of the projection type in the bounds,
diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs
index 85958c417..a6ba74220 100644
--- a/compiler/rustc_lint/src/types.rs
+++ b/compiler/rustc_lint/src/types.rs
@@ -4,7 +4,8 @@ use crate::{
         AtomicOrderingFence, AtomicOrderingLoad, AtomicOrderingStore, ImproperCTypes,
         InvalidAtomicOrderingDiag, OnlyCastu8ToChar, OverflowingBinHex, OverflowingBinHexSign,
         OverflowingBinHexSub, OverflowingInt, OverflowingIntHelp, OverflowingLiteral,
-        OverflowingUInt, RangeEndpointOutOfRange, UnusedComparisons, VariantSizeDifferencesDiag,
+        OverflowingUInt, RangeEndpointOutOfRange, UnusedComparisons, UseInclusiveRange,
+        VariantSizeDifferencesDiag,
     },
 };
 use crate::{LateContext, LateLintPass, LintContext};
@@ -17,7 +18,7 @@ use rustc_hir::{is_range_literal, Expr, ExprKind, Node};
 use rustc_middle::ty::layout::{IntegerExt, LayoutOf, SizeSkeleton};
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{
-    self, AdtKind, DefIdTree, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
+    self, AdtKind, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
 };
 use rustc_span::def_id::LocalDefId;
 use rustc_span::source_map;
@@ -136,6 +137,14 @@ fn lint_overflowing_range_endpoint<'tcx>(
     expr: &'tcx hir::Expr<'tcx>,
     ty: &str,
 ) -> bool {
+    // Look past casts to support cases like `0..256 as u8`
+    let (expr, lit_span) = if let Node::Expr(par_expr) = cx.tcx.hir().get(cx.tcx.hir().parent_id(expr.hir_id))
+      && let ExprKind::Cast(_, _) = par_expr.kind {
+        (par_expr, expr.span)
+    } else {
+        (expr, expr.span)
+    };
+
     // We only want to handle exclusive (`..`) ranges,
     // which are represented as `ExprKind::Struct`.
     let par_id = cx.tcx.hir().parent_id(expr.hir_id);
@@ -155,7 +164,6 @@ fn lint_overflowing_range_endpoint<'tcx>(
     if !(eps[1].expr.hir_id == expr.hir_id && lit_val - 1 == max) {
         return false;
     };
-    let Ok(start) = cx.sess().source_map().span_to_snippet(eps[0].span) else { return false };
 
     use rustc_ast::{LitIntType, LitKind};
     let suffix = match lit.node {
@@ -164,16 +172,28 @@ fn lint_overflowing_range_endpoint<'tcx>(
         LitKind::Int(_, LitIntType::Unsuffixed) => "",
         _ => bug!(),
     };
-    cx.emit_spanned_lint(
-        OVERFLOWING_LITERALS,
-        struct_expr.span,
-        RangeEndpointOutOfRange {
-            ty,
-            suggestion: struct_expr.span,
+
+    let sub_sugg = if expr.span.lo() == lit_span.lo() {
+        let Ok(start) = cx.sess().source_map().span_to_snippet(eps[0].span) else { return false };
+        UseInclusiveRange::WithoutParen {
+            sugg: struct_expr.span.shrink_to_lo().to(lit_span.shrink_to_hi()),
             start,
             literal: lit_val - 1,
             suffix,
-        },
+        }
+    } else {
+        UseInclusiveRange::WithParen {
+            eq_sugg: expr.span.shrink_to_lo(),
+            lit_sugg: lit_span,
+            literal: lit_val - 1,
+            suffix,
+        }
+    };
+
+    cx.emit_spanned_lint(
+        OVERFLOWING_LITERALS,
+        struct_expr.span,
+        RangeEndpointOutOfRange { ty, sub: sub_sugg },
     );
 
     // We've just emitted a lint, special cased for `(...)..MAX+1` ranges,
@@ -750,7 +770,7 @@ pub(crate) fn repr_nullable_ptr<'tcx>(
     debug!("is_repr_nullable_ptr(cx, ty = {:?})", ty);
     if let ty::Adt(ty_def, substs) = ty.kind() {
         let field_ty = match &ty_def.variants().raw[..] {
-            [var_one, var_two] => match (&var_one.fields[..], &var_two.fields[..]) {
+            [var_one, var_two] => match (&var_one.fields.raw[..], &var_two.fields.raw[..]) {
                 ([], [field]) | ([field], []) => field.ty(cx.tcx, substs),
                 _ => return None,
             },
diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index 3a92f5806..28cc63198 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -12,9 +12,9 @@ use rustc_errors::{pluralize, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::DefId;
-use rustc_infer::traits::util::elaborate_predicates_with_span;
+use rustc_infer::traits::util::elaborate;
 use rustc_middle::ty::adjustment;
-use rustc_middle::ty::{self, DefIdTree, Ty};
+use rustc_middle::ty::{self, Ty};
 use rustc_span::symbol::Symbol;
 use rustc_span::symbol::{kw, sym};
 use rustc_span::{BytePos, Span};
@@ -103,8 +103,10 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
             && let ty = cx.typeck_results().expr_ty(&await_expr)
             && let ty::Alias(ty::Opaque, ty::AliasTy { def_id: future_def_id, .. }) = ty.kind()
             && cx.tcx.ty_is_opaque_future(ty)
-            // FIXME: This also includes non-async fns that return `impl Future`.
             && let async_fn_def_id = cx.tcx.parent(*future_def_id)
+            && matches!(cx.tcx.def_kind(async_fn_def_id), DefKind::Fn | DefKind::AssocFn)
+            // Check that this `impl Future` actually comes from an `async fn`
+            && cx.tcx.asyncness(async_fn_def_id).is_async()
             && check_must_use_def(
                 cx,
                 async_fn_def_id,
@@ -123,7 +125,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
         let must_use_result = is_ty_must_use(cx, ty, &expr, expr.span);
         let type_lint_emitted_or_suppressed = match must_use_result {
             Some(path) => {
-                emit_must_use_untranslated(cx, &path, "", "", 1);
+                emit_must_use_untranslated(cx, &path, "", "", 1, false);
                 true
             }
             None => false,
@@ -254,24 +256,23 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                 }
                 ty::Adt(def, _) => is_def_must_use(cx, def.did(), span),
                 ty::Alias(ty::Opaque, ty::AliasTy { def_id: def, .. }) => {
-                    elaborate_predicates_with_span(
-                        cx.tcx,
-                        cx.tcx.explicit_item_bounds(def).iter().cloned(),
-                    )
-                    .find_map(|obligation| {
-                        // We only look at the `DefId`, so it is safe to skip the binder here.
-                        if let ty::PredicateKind::Clause(ty::Clause::Trait(
-                            ref poly_trait_predicate,
-                        )) = obligation.predicate.kind().skip_binder()
-                        {
-                            let def_id = poly_trait_predicate.trait_ref.def_id;
-
-                            is_def_must_use(cx, def_id, span)
-                        } else {
-                            None
-                        }
-                    })
-                    .map(|inner| MustUsePath::Opaque(Box::new(inner)))
+                    elaborate(cx.tcx, cx.tcx.explicit_item_bounds(def).iter().cloned())
+                        // We only care about self bounds for the impl-trait
+                        .filter_only_self()
+                        .find_map(|(pred, _span)| {
+                            // We only look at the `DefId`, so it is safe to skip the binder here.
+                            if let ty::PredicateKind::Clause(ty::Clause::Trait(
+                                ref poly_trait_predicate,
+                            )) = pred.kind().skip_binder()
+                            {
+                                let def_id = poly_trait_predicate.trait_ref.def_id;
+
+                                is_def_must_use(cx, def_id, span)
+                            } else {
+                                None
+                            }
+                        })
+                        .map(|inner| MustUsePath::Opaque(Box::new(inner)))
                 }
                 ty::Dynamic(binders, _, _) => binders.iter().find_map(|predicate| {
                     if let ty::ExistentialPredicate::Trait(ref trait_ref) = predicate.skip_binder()
@@ -358,6 +359,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                         descr_pre_path,
                         descr_post_path,
                         1,
+                        false,
                     )
                 })
                 .is_some()
@@ -370,6 +372,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
             descr_pre: &str,
             descr_post: &str,
             plural_len: usize,
+            is_inner: bool,
         ) {
             let plural_suffix = pluralize!(plural_len);
 
@@ -377,20 +380,22 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                 MustUsePath::Suppressed => {}
                 MustUsePath::Boxed(path) => {
                     let descr_pre = &format!("{}boxed ", descr_pre);
-                    emit_must_use_untranslated(cx, path, descr_pre, descr_post, plural_len);
+                    emit_must_use_untranslated(cx, path, descr_pre, descr_post, plural_len, true);
                 }
                 MustUsePath::Opaque(path) => {
                     let descr_pre = &format!("{}implementer{} of ", descr_pre, plural_suffix);
-                    emit_must_use_untranslated(cx, path, descr_pre, descr_post, plural_len);
+                    emit_must_use_untranslated(cx, path, descr_pre, descr_post, plural_len, true);
                 }
                 MustUsePath::TraitObject(path) => {
                     let descr_post = &format!(" trait object{}{}", plural_suffix, descr_post);
-                    emit_must_use_untranslated(cx, path, descr_pre, descr_post, plural_len);
+                    emit_must_use_untranslated(cx, path, descr_pre, descr_post, plural_len, true);
                 }
                 MustUsePath::TupleElement(elems) => {
                     for (index, path) in elems {
                         let descr_post = &format!(" in tuple element {}", index);
-                        emit_must_use_untranslated(cx, path, descr_pre, descr_post, plural_len);
+                        emit_must_use_untranslated(
+                            cx, path, descr_pre, descr_post, plural_len, true,
+                        );
                     }
                 }
                 MustUsePath::Array(path, len) => {
@@ -401,6 +406,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                         descr_pre,
                         descr_post,
                         plural_len.saturating_add(usize::try_from(*len).unwrap_or(usize::MAX)),
+                        true,
                     );
                 }
                 MustUsePath::Closure(span) => {
@@ -418,19 +424,6 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                     );
                 }
                 MustUsePath::Def(span, def_id, reason) => {
-                    let suggestion = if matches!(
-                        cx.tcx.get_diagnostic_name(*def_id),
-                        Some(sym::add)
-                            | Some(sym::sub)
-                            | Some(sym::mul)
-                            | Some(sym::div)
-                            | Some(sym::rem)
-                            | Some(sym::neg),
-                    ) {
-                        Some(UnusedDefSuggestion::Default { span: span.shrink_to_lo() })
-                    } else {
-                        None
-                    };
                     cx.emit_spanned_lint(
                         UNUSED_MUST_USE,
                         *span,
@@ -440,7 +433,8 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                             cx,
                             def_id: *def_id,
                             note: *reason,
-                            suggestion,
+                            suggestion: (!is_inner)
+                                .then_some(UnusedDefSuggestion { span: span.shrink_to_lo() }),
                         },
                     );
                 }
@@ -812,7 +806,9 @@ trait UnusedDelimLint {
     fn check_item(&mut self, cx: &EarlyContext<'_>, item: &ast::Item) {
         use ast::ItemKind::*;
 
-        if let Const(.., Some(expr)) | Static(.., Some(expr)) = &item.kind {
+        if let Const(box ast::ConstItem { expr: Some(expr), .. })
+        | Static(box ast::StaticItem { expr: Some(expr), .. }) = &item.kind
+        {
             self.check_unused_delims_expr(
                 cx,
                 expr,
@@ -1349,9 +1345,8 @@ declare_lint! {
     /// ### Example
     ///
     /// ```rust
-    /// #![feature(box_syntax)]
     /// fn main() {
-    ///     let a = (box [1, 2, 3]).len();
+    ///     let a = Box::new([1, 2, 3]).len();
     /// }
     /// ```
     ///
@@ -1372,7 +1367,11 @@ declare_lint_pass!(UnusedAllocation => [UNUSED_ALLOCATION]);
 impl<'tcx> LateLintPass<'tcx> for UnusedAllocation {
     fn check_expr(&mut self, cx: &LateContext<'_>, e: &hir::Expr<'_>) {
         match e.kind {
-            hir::ExprKind::Box(_) => {}
+            hir::ExprKind::Call(path_expr, [_])
+                if let hir::ExprKind::Path(qpath) = &path_expr.kind
+                && let Some(did) = cx.qpath_res(qpath, path_expr.hir_id).opt_def_id()
+                && cx.tcx.is_diagnostic_item(sym::box_new, did)
+                => {}
             _ => return,
         }
 
diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index 46ec1a2dc..9d6ab0b75 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -1026,12 +1026,13 @@ declare_lint! {
     /// ### Example
     ///
     /// ```rust,compile_fail
-    /// #![feature(const_ptr_read)]
+    /// #![feature(const_mut_refs)]
     /// const FOO: () = unsafe {
     ///     let x = &[0_u8; 4];
     ///     let y = x.as_ptr().cast::<u32>();
-    ///     y.read(); // the address of a `u8` array is unknown and thus we don't know if
-    ///     // it is aligned enough for reading a `u32`.
+    ///     let mut z = 123;
+    ///     y.copy_to_nonoverlapping(&mut z, 1); // the address of a `u8` array is unknown
+    ///     // and thus we don't know if it is aligned enough for copying a `u32`.
     /// };
     /// ```
     ///
@@ -3229,6 +3230,45 @@ declare_lint! {
     };
 }
 
+declare_lint! {
+    /// The `ambiguous_glob_reexports` lint detects cases where names re-exported via globs
+    /// collide. Downstream users trying to use the same name re-exported from multiple globs
+    /// will receive a warning pointing out redefinition of the same name.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,compile_fail
+    /// #![deny(ambiguous_glob_reexports)]
+    /// pub mod foo {
+    ///     pub type X = u8;
+    /// }
+    ///
+    /// pub mod bar {
+    ///     pub type Y = u8;
+    ///     pub type X = u8;
+    /// }
+    ///
+    /// pub use foo::*;
+    /// pub use bar::*;
+    ///
+    ///
+    /// pub fn main() {}
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// This was previously accepted but it could silently break a crate's downstream users code.
+    /// For example, if `foo::*` and `bar::*` were re-exported before `bar::X` was added to the
+    /// re-exports, down stream users could use `this_crate::X` without problems. However, adding
+    /// `bar::X` would cause compilation errors in downstream crates because `X` is defined
+    /// multiple times in the same namespace of `this_crate`.
+    pub AMBIGUOUS_GLOB_REEXPORTS,
+    Warn,
+    "ambiguous glob re-exports",
+}
+
 declare_lint_pass! {
     /// Does nothing as a lint pass, but registers some `Lint`s
     /// that are used by other parts of the compiler.
@@ -3336,6 +3376,7 @@ declare_lint_pass! {
         NAMED_ARGUMENTS_USED_POSITIONALLY,
         IMPLIED_BOUNDS_ENTAILMENT,
         BYTE_SLICE_IN_PACKED_STRUCT_WITH_DERIVE,
+        AMBIGUOUS_GLOB_REEXPORTS,
     ]
 }
 
@@ -3967,14 +4008,9 @@ declare_lint! {
     ///
     /// ### Example
     ///
-    /// ```rust,ignore (need FFI)
-    /// #![feature(ffi_unwind_calls)]
+    /// ```rust
     /// #![feature(c_unwind)]
-    ///
-    /// # mod impl {
-    /// #     #[no_mangle]
-    /// #     pub fn "C-unwind" fn foo() {}
-    /// # }
+    /// #![warn(ffi_unwind_calls)]
     ///
     /// extern "C-unwind" {
     ///     fn foo();
diff --git a/compiler/rustc_lint_defs/src/lib.rs b/compiler/rustc_lint_defs/src/lib.rs
index 534aff7fb..7ea472ed5 100644
--- a/compiler/rustc_lint_defs/src/lib.rs
+++ b/compiler/rustc_lint_defs/src/lib.rs
@@ -8,7 +8,7 @@ extern crate rustc_macros;
 pub use self::Level::*;
 use rustc_ast::node_id::NodeId;
 use rustc_ast::{AttrId, Attribute};
-use rustc_data_structures::fx::FxIndexMap;
+use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey};
 use rustc_error_messages::{DiagnosticMessage, MultiSpan};
 use rustc_hir::HashStableContext;
@@ -529,10 +529,21 @@ pub enum BuiltinLintDiagnostics {
         vis_span: Span,
         ident_span: Span,
     },
+    AmbiguousGlobReexports {
+        /// The name for which collision(s) have occurred.
+        name: String,
+        /// The name space for which the collision(s) occurred in.
+        namespace: String,
+        /// Span where the name is first re-exported.
+        first_reexport_span: Span,
+        /// Span where the same name is also re-exported.
+        duplicate_reexport_span: Span,
+    },
 }
 
 /// Lints that are buffered up early on in the `Session` before the
 /// `LintLevels` is calculated.
+#[derive(Debug)]
 pub struct BufferedEarlyLint {
     /// The span of code that we are linting on.
     pub span: MultiSpan,
@@ -551,7 +562,7 @@ pub struct BufferedEarlyLint {
     pub diagnostic: BuiltinLintDiagnostics,
 }
 
-#[derive(Default)]
+#[derive(Default, Debug)]
 pub struct LintBuffer {
     pub map: FxIndexMap<NodeId, Vec<BufferedEarlyLint>>,
 }
@@ -601,6 +612,8 @@ impl LintBuffer {
     }
 }
 
+pub type RegisteredTools = FxIndexSet<Ident>;
+
 /// Declares a static item of type `&'static Lint`.
 ///
 /// See <https://rustc-dev-guide.rust-lang.org/diagnostics.html> for
diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
index 9fe59a1d8..b0783d75d 100644
--- a/compiler/rustc_llvm/build.rs
+++ b/compiler/rustc_llvm/build.rs
@@ -10,6 +10,7 @@ const OPTIONAL_COMPONENTS: &[&str] = &[
     "aarch64",
     "amdgpu",
     "avr",
+    "loongarch",
     "m68k",
     "mips",
     "powerpc",
@@ -333,6 +334,7 @@ fn main() {
     } else if target.contains("darwin")
         || target.contains("freebsd")
         || target.contains("windows-gnullvm")
+        || target.contains("aix")
     {
         "c++"
     } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() {
diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
index 4761ce83f..08e38b0c9 100644
--- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
@@ -26,7 +26,6 @@
 #endif
 #include "llvm/Support/Host.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
 #include "llvm/Transforms/IPO/FunctionImport.h"
 #include "llvm/Transforms/IPO/Internalize.h"
@@ -35,7 +34,6 @@
 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Bitcode/BitcodeWriter.h"
-#include "llvm-c/Transforms/PassManagerBuilder.h"
 
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
@@ -148,6 +146,12 @@ extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
 #define SUBTARGET_HEXAGON
 #endif
 
+#ifdef LLVM_COMPONENT_LOONGARCH
+#define SUBTARGET_LOONGARCH SUBTARGET(LoongArch)
+#else
+#define SUBTARGET_LOONGARCH
+#endif
+
 #define GEN_SUBTARGETS                                                         \
   SUBTARGET_X86                                                                \
   SUBTARGET_ARM                                                                \
@@ -161,6 +165,7 @@ extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
   SUBTARGET_SPARC                                                              \
   SUBTARGET_HEXAGON                                                            \
   SUBTARGET_RISCV                                                              \
+  SUBTARGET_LOONGARCH                                                          \
 
 #define SUBTARGET(x)                                                           \
   namespace llvm {                                                             \
@@ -370,7 +375,8 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     bool EmitStackSizeSection,
     bool RelaxELFRelocations,
     bool UseInitArray,
-    const char *SplitDwarfFile) {
+    const char *SplitDwarfFile,
+    bool ForceEmulatedTls) {
 
   auto OptLevel = fromRust(RustOptLevel);
   auto RM = fromRust(RustReloc);
@@ -402,6 +408,10 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   }
   Options.RelaxELFRelocations = RelaxELFRelocations;
   Options.UseInitArray = UseInitArray;
+  if (ForceEmulatedTls) {
+    Options.ExplicitEmulatedTLS = true;
+    Options.EmulatedTLS = true;
+  }
 
   if (TrapUnreachable) {
     // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
@@ -1160,13 +1170,6 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
   // Otherwise, we sometimes lose `static` values -- see #60184.
   computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
                                   deadIsPrevailing, /* ImportEnabled = */ false);
-  ComputeCrossModuleImport(
-    Ret->Index,
-    Ret->ModuleToDefinedGVSummaries,
-    Ret->ImportLists,
-    Ret->ExportLists
-  );
-
   // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
   // impacts the caching.
   //
@@ -1183,6 +1186,16 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
       return true;
     return Prevailing->second == S;
   };
+  ComputeCrossModuleImport(
+    Ret->Index,
+    Ret->ModuleToDefinedGVSummaries,
+#if LLVM_VERSION_GE(17, 0)
+    isPrevailing,
+#endif
+    Ret->ImportLists,
+    Ret->ExportLists
+  );
+
   auto recordNewLinkage = [&](StringRef ModuleIdentifier,
                               GlobalValue::GUID GUID,
                               GlobalValue::LinkageTypes NewLinkage) {
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
index e3493caaa..49acd71b3 100644
--- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -152,10 +152,6 @@ LLVMRustInsertPrivateGlobal(LLVMModuleRef M, LLVMTypeRef Ty) {
                                  nullptr));
 }
 
-extern "C" LLVMTypeRef LLVMRustMetadataTypeInContext(LLVMContextRef C) {
-  return wrap(Type::getMetadataTy(*unwrap(C)));
-}
-
 static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) {
   switch (Kind) {
   case AlwaysInline:
@@ -480,11 +476,6 @@ extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints,
 #endif
 }
 
-extern "C" void LLVMRustAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm,
-                                              size_t AsmLen) {
-  unwrap(M)->appendModuleInlineAsm(StringRef(Asm, AsmLen));
-}
-
 typedef DIBuilder *LLVMRustDIBuilderRef;
 
 template <typename DIT> DIT *unwrapDIPtr(LLVMMetadataRef Ref) {
@@ -682,6 +673,7 @@ enum class LLVMRustDebugEmissionKind {
   NoDebug,
   FullDebug,
   LineTablesOnly,
+  DebugDirectivesOnly,
 };
 
 static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) {
@@ -692,6 +684,8 @@ static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind)
     return DICompileUnit::DebugEmissionKind::FullDebug;
   case LLVMRustDebugEmissionKind::LineTablesOnly:
     return DICompileUnit::DebugEmissionKind::LineTablesOnly;
+  case LLVMRustDebugEmissionKind::DebugDirectivesOnly:
+    return DICompileUnit::DebugEmissionKind::DebugDirectivesOnly;
   default:
     report_fatal_error("bad DebugEmissionKind.");
   }
@@ -750,10 +744,6 @@ extern "C" bool LLVMRustHasModuleFlag(LLVMModuleRef M, const char *Name,
   return unwrap(M)->getModuleFlag(StringRef(Name, Len)) != nullptr;
 }
 
-extern "C" LLVMValueRef LLVMRustMetadataAsValue(LLVMContextRef C, LLVMMetadataRef MD) {
-  return wrap(MetadataAsValue::get(*unwrap(C), unwrap(MD)));
-}
-
 extern "C" void LLVMRustGlobalAddMetadata(
     LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) {
   unwrap<GlobalObject>(Global)->addMetadata(Kind, *unwrap<MDNode>(MD));
@@ -841,6 +831,28 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction(
   return wrap(Sub);
 }
 
+extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMethod(
+    LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope,
+    const char *Name, size_t NameLen,
+    const char *LinkageName, size_t LinkageNameLen,
+    LLVMMetadataRef File, unsigned LineNo,
+    LLVMMetadataRef Ty, LLVMRustDIFlags Flags,
+    LLVMRustDISPFlags SPFlags, LLVMMetadataRef TParam) {
+  DITemplateParameterArray TParams =
+      DITemplateParameterArray(unwrap<MDTuple>(TParam));
+  DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags);
+  DINode::DIFlags llvmFlags = fromRust(Flags);
+  DISubprogram *Sub = Builder->createMethod(
+      unwrapDI<DIScope>(Scope),
+      StringRef(Name, NameLen),
+      StringRef(LinkageName, LinkageNameLen),
+      unwrapDI<DIFile>(File), LineNo,
+      unwrapDI<DISubroutineType>(Ty),
+      0, 0, nullptr, // VTable params aren't used
+      llvmFlags, llvmSPFlags, TParams);
+  return wrap(Sub);
+}
+
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateBasicType(
     LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen,
     uint64_t SizeInBits, unsigned Encoding) {
@@ -1150,6 +1162,8 @@ extern "C" void LLVMRustWriteValueToString(LLVMValueRef V,
 }
 
 // LLVMArrayType function does not support 64-bit ElementCount
+// FIXME: replace with LLVMArrayType2 when bumped minimal version to llvm-17
+// https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc
 extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy,
                                          uint64_t ElementCount) {
   return wrap(ArrayType::get(unwrap(ElementTy), ElementCount));
@@ -1405,61 +1419,6 @@ extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef,
   return true;
 }
 
-extern "C" LLVMValueRef LLVMRustBuildCleanupPad(LLVMBuilderRef B,
-                                                LLVMValueRef ParentPad,
-                                                unsigned ArgCount,
-                                                LLVMValueRef *LLArgs,
-                                                const char *Name) {
-  Value **Args = unwrap(LLArgs);
-  if (ParentPad == nullptr) {
-    Type *Ty = Type::getTokenTy(unwrap(B)->getContext());
-    ParentPad = wrap(Constant::getNullValue(Ty));
-  }
-  return wrap(unwrap(B)->CreateCleanupPad(
-      unwrap(ParentPad), ArrayRef<Value *>(Args, ArgCount), Name));
-}
-
-extern "C" LLVMValueRef LLVMRustBuildCleanupRet(LLVMBuilderRef B,
-                                                LLVMValueRef CleanupPad,
-                                                LLVMBasicBlockRef UnwindBB) {
-  CleanupPadInst *Inst = cast<CleanupPadInst>(unwrap(CleanupPad));
-  return wrap(unwrap(B)->CreateCleanupRet(Inst, unwrap(UnwindBB)));
-}
-
-extern "C" LLVMValueRef
-LLVMRustBuildCatchPad(LLVMBuilderRef B, LLVMValueRef ParentPad,
-                      unsigned ArgCount, LLVMValueRef *LLArgs, const char *Name) {
-  Value **Args = unwrap(LLArgs);
-  return wrap(unwrap(B)->CreateCatchPad(
-      unwrap(ParentPad), ArrayRef<Value *>(Args, ArgCount), Name));
-}
-
-extern "C" LLVMValueRef LLVMRustBuildCatchRet(LLVMBuilderRef B,
-                                              LLVMValueRef Pad,
-                                              LLVMBasicBlockRef BB) {
-  return wrap(unwrap(B)->CreateCatchRet(cast<CatchPadInst>(unwrap(Pad)),
-                                              unwrap(BB)));
-}
-
-extern "C" LLVMValueRef LLVMRustBuildCatchSwitch(LLVMBuilderRef B,
-                                                 LLVMValueRef ParentPad,
-                                                 LLVMBasicBlockRef BB,
-                                                 unsigned NumHandlers,
-                                                 const char *Name) {
-  if (ParentPad == nullptr) {
-    Type *Ty = Type::getTokenTy(unwrap(B)->getContext());
-    ParentPad = wrap(Constant::getNullValue(Ty));
-  }
-  return wrap(unwrap(B)->CreateCatchSwitch(unwrap(ParentPad), unwrap(BB),
-                                                 NumHandlers, Name));
-}
-
-extern "C" void LLVMRustAddHandler(LLVMValueRef CatchSwitchRef,
-                                   LLVMBasicBlockRef Handler) {
-  Value *CatchSwitch = unwrap(CatchSwitchRef);
-  cast<CatchSwitchInst>(CatchSwitch)->addHandler(unwrap(Handler));
-}
-
 extern "C" OperandBundleDef *LLVMRustBuildOperandBundleDef(const char *Name,
                                                            LLVMValueRef *Inputs,
                                                            unsigned NumInputs) {
@@ -1624,6 +1583,7 @@ extern "C" void LLVMRustSetLinkage(LLVMValueRef V,
   LLVMSetLinkage(V, fromRust(RustLinkage));
 }
 
+// FIXME: replace with LLVMConstInBoundsGEP2 when bumped minimal version to llvm-14
 extern "C" LLVMValueRef LLVMRustConstInBoundsGEP2(LLVMTypeRef Ty,
                                                   LLVMValueRef ConstantVal,
                                                   LLVMValueRef *ConstantIndices,
@@ -1701,12 +1661,6 @@ extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) {
   return toRust(LLVMGetVisibility(V));
 }
 
-// Oh hey, a binding that makes sense for once? (because LLVM’s own do not)
-extern "C" LLVMValueRef LLVMRustBuildIntCast(LLVMBuilderRef B, LLVMValueRef Val,
-                                             LLVMTypeRef DestTy, bool isSigned) {
-  return wrap(unwrap(B)->CreateIntCast(unwrap(Val), unwrap(DestTy), isSigned, ""));
-}
-
 extern "C" void LLVMRustSetVisibility(LLVMValueRef V,
                                       LLVMRustVisibility RustVisibility) {
   LLVMSetVisibility(V, fromRust(RustVisibility));
diff --git a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp
index 974207e91..0493d6b05 100644
--- a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp
@@ -9,7 +9,6 @@
 
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Object/ObjectFile.h"
-#include "llvm/ADT/Optional.h"
 
 using namespace llvm;
 using namespace llvm::sys;
diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs
index 8542dcf5b..a49ded4fd 100644
--- a/compiler/rustc_llvm/src/lib.rs
+++ b/compiler/rustc_llvm/src/lib.rs
@@ -30,7 +30,7 @@ pub unsafe extern "C" fn LLVMRustStringWriteImpl(
     ptr: *const c_char,
     size: size_t,
 ) {
-    let slice = slice::from_raw_parts(ptr as *const u8, size as usize);
+    let slice = slice::from_raw_parts(ptr as *const u8, size);
 
     sr.bytes.borrow_mut().extend_from_slice(slice);
 }
@@ -103,6 +103,14 @@ pub fn initialize_available_targets() {
         LLVMInitializeM68kAsmParser
     );
     init_target!(
+        llvm_component = "loongarch",
+        LLVMInitializeLoongArchTargetInfo,
+        LLVMInitializeLoongArchTarget,
+        LLVMInitializeLoongArchTargetMC,
+        LLVMInitializeLoongArchAsmPrinter,
+        LLVMInitializeLoongArchAsmParser
+    );
+    init_target!(
         llvm_component = "mips",
         LLVMInitializeMipsTargetInfo,
         LLVMInitializeMipsTarget,
diff --git a/compiler/rustc_log/src/lib.rs b/compiler/rustc_log/src/lib.rs
index 22924efa9..3cbb2c21e 100644
--- a/compiler/rustc_log/src/lib.rs
+++ b/compiler/rustc_log/src/lib.rs
@@ -40,7 +40,6 @@
 
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
-#![feature(is_terminal)]
 
 use std::env::{self, VarError};
 use std::fmt::{self, Display};
@@ -83,7 +82,7 @@ pub fn init_env_logger(env: &str) -> Result<(), Error> {
         .with_verbose_exit(verbose_entry_exit)
         .with_verbose_entry(verbose_entry_exit)
         .with_indent_amount(2);
-    #[cfg(parallel_compiler)]
+    #[cfg(all(parallel_compiler, debug_assertions))]
     let layer = layer.with_thread_ids(true).with_thread_names(true);
 
     let subscriber = tracing_subscriber::Registry::default().with(filter).with(layer);
diff --git a/compiler/rustc_macros/Cargo.toml b/compiler/rustc_macros/Cargo.toml
index 547c8debb..745983e7e 100644
--- a/compiler/rustc_macros/Cargo.toml
+++ b/compiler/rustc_macros/Cargo.toml
@@ -10,8 +10,8 @@ proc-macro = true
 annotate-snippets = "0.9"
 fluent-bundle = "0.15.2"
 fluent-syntax = "0.11"
-synstructure = "0.12.1"
-syn = { version = "1", features = ["full"] }
+synstructure = "0.13.0"
+syn = { version = "2", features = ["full"] }
 proc-macro2 = "1"
 quote = "1"
 unic-langid = { version = "0.9.0", features = ["macros"] }
diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
index 46068f8c8..427c82c41 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
@@ -1,19 +1,17 @@
 #![deny(unused_must_use)]
 
 use crate::diagnostics::error::{
-    invalid_nested_attr, span_err, throw_invalid_attr, throw_invalid_nested_attr, throw_span_err,
-    DiagnosticDeriveError,
+    span_err, throw_invalid_attr, throw_span_err, DiagnosticDeriveError,
 };
 use crate::diagnostics::utils::{
     build_field_mapping, is_doc_comment, report_error_if_not_applied_to_span, report_type_error,
-    should_generate_set_arg, type_is_unit, type_matches_path, FieldInfo, FieldInnerTy, FieldMap,
-    HasFieldMap, SetOnce, SpannedOption, SubdiagnosticKind,
+    should_generate_set_arg, type_is_bool, type_is_unit, type_matches_path, FieldInfo,
+    FieldInnerTy, FieldMap, HasFieldMap, SetOnce, SpannedOption, SubdiagnosticKind,
 };
 use proc_macro2::{Ident, Span, TokenStream};
 use quote::{format_ident, quote};
-use syn::{
-    parse_quote, spanned::Spanned, Attribute, Meta, MetaList, MetaNameValue, NestedMeta, Path, Type,
-};
+use syn::Token;
+use syn::{parse_quote, spanned::Spanned, Attribute, Meta, Path, Type};
 use synstructure::{BindingInfo, Structure, VariantInfo};
 
 /// What kind of diagnostic is being derived - a fatal/error/warning or a lint?
@@ -77,7 +75,7 @@ impl DiagnosticDeriveBuilder {
         match ast.data {
             syn::Data::Struct(..) | syn::Data::Enum(..) => (),
             syn::Data::Union(..) => {
-                span_err(span, "diagnostic derives can only be used on structs and enums");
+                span_err(span, "diagnostic derives can only be used on structs and enums").emit();
             }
         }
 
@@ -121,7 +119,7 @@ impl DiagnosticDeriveBuilder {
 impl<'a> DiagnosticDeriveVariantBuilder<'a> {
     /// Generates calls to `code` and similar functions based on the attributes on the type or
     /// variant.
-    pub fn preamble<'s>(&mut self, variant: &VariantInfo<'s>) -> TokenStream {
+    pub fn preamble(&mut self, variant: &VariantInfo<'_>) -> TokenStream {
         let ast = variant.ast();
         let attrs = &ast.attrs;
         let preamble = attrs.iter().map(|attr| {
@@ -135,7 +133,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
 
     /// Generates calls to `span_label` and similar functions based on the attributes on fields or
     /// calls to `set_arg` when no attributes are present.
-    pub fn body<'s>(&mut self, variant: &VariantInfo<'s>) -> TokenStream {
+    pub fn body(&mut self, variant: &VariantInfo<'_>) -> TokenStream {
         let mut body = quote! {};
         // Generate `set_arg` calls first..
         for binding in variant.bindings().iter().filter(|bi| should_generate_set_arg(bi.ast())) {
@@ -160,8 +158,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
         };
 
         if let SubdiagnosticKind::MultipartSuggestion { .. } = subdiag {
-            let meta = attr.parse_meta()?;
-            throw_invalid_attr!(attr, &meta, |diag| diag
+            throw_invalid_attr!(attr, |diag| diag
                 .help("consider creating a `Subdiagnostic` instead"));
         }
 
@@ -191,71 +188,44 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
             return Ok(quote! {});
         }
 
-        let name = attr.path.segments.last().unwrap().ident.to_string();
+        let name = attr.path().segments.last().unwrap().ident.to_string();
         let name = name.as_str();
-        let meta = attr.parse_meta()?;
 
-        if name == "diag" {
-            let Meta::List(MetaList { ref nested, .. }) = meta else {
-                throw_invalid_attr!(
-                    attr,
-                    &meta
-                );
-            };
+        let mut first = true;
 
-            let mut nested_iter = nested.into_iter().peekable();
+        if name == "diag" {
+            let mut tokens = TokenStream::new();
+            attr.parse_nested_meta(|nested| {
+                let path = &nested.path;
 
-            match nested_iter.peek() {
-                Some(NestedMeta::Meta(Meta::Path(slug))) => {
-                    self.slug.set_once(slug.clone(), slug.span().unwrap());
-                    nested_iter.next();
+                if first && (nested.input.is_empty() || nested.input.peek(Token![,])) {
+                    self.slug.set_once(path.clone(), path.span().unwrap());
+                    first = false;
+                    return Ok(())
                 }
-                Some(NestedMeta::Meta(Meta::NameValue { .. })) => {}
-                Some(nested_attr) => throw_invalid_nested_attr!(attr, nested_attr, |diag| diag
-                    .help("a diagnostic slug is required as the first argument")),
-                None => throw_invalid_attr!(attr, &meta, |diag| diag
-                    .help("a diagnostic slug is required as the first argument")),
-            };
 
-            // Remaining attributes are optional, only `code = ".."` at the moment.
-            let mut tokens = TokenStream::new();
-            for nested_attr in nested_iter {
-                let (value, path) = match nested_attr {
-                    NestedMeta::Meta(Meta::NameValue(MetaNameValue {
-                        lit: syn::Lit::Str(value),
-                        path,
-                        ..
-                    })) => (value, path),
-                    NestedMeta::Meta(Meta::Path(_)) => {
-                        invalid_nested_attr(attr, nested_attr)
-                            .help("diagnostic slug must be the first argument")
-                            .emit();
-                        continue;
-                    }
-                    _ => {
-                        invalid_nested_attr(attr, nested_attr).emit();
-                        continue;
-                    }
+                first = false;
+
+                let Ok(nested) = nested.value() else {
+                    span_err(nested.input.span().unwrap(), "diagnostic slug must be the first argument").emit();
+                    return Ok(())
                 };
 
-                let nested_name = path.segments.last().unwrap().ident.to_string();
-                // Struct attributes are only allowed to be applied once, and the diagnostic
-                // changes will be set in the initialisation code.
-                let span = value.span().unwrap();
-                match nested_name.as_str() {
-                    "code" => {
-                        self.code.set_once((), span);
-
-                        let code = value.value();
-                        tokens.extend(quote! {
-                            #diag.code(rustc_errors::DiagnosticId::Error(#code.to_string()));
-                        });
-                    }
-                    _ => invalid_nested_attr(attr, nested_attr)
-                        .help("only `code` is a valid nested attributes following the slug")
-                        .emit(),
+                if path.is_ident("code") {
+                    self.code.set_once((), path.span().unwrap());
+
+                    let code = nested.parse::<syn::LitStr>()?;
+                    tokens.extend(quote! {
+                        #diag.code(rustc_errors::DiagnosticId::Error(#code.to_string()));
+                    });
+                } else {
+                    span_err(path.span().unwrap(), "unknown argument").note("only the `code` parameter is valid after the slug").emit();
+
+                    // consume the buffer so we don't have syntax errors from syn
+                    let _ = nested.parse::<TokenStream>();
                 }
-            }
+                Ok(())
+            })?;
             return Ok(tokens);
         }
 
@@ -270,7 +240,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                 Ok(self.add_subdiagnostic(&fn_ident, slug))
             }
             SubdiagnosticKind::Label | SubdiagnosticKind::Suggestion { .. } => {
-                throw_invalid_attr!(attr, &meta, |diag| diag
+                throw_invalid_attr!(attr, |diag| diag
                     .help("`#[label]` and `#[suggestion]` can only be applied to fields"));
             }
             SubdiagnosticKind::MultipartSuggestion { .. } => unreachable!(),
@@ -309,7 +279,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                     return quote! {};
                 }
 
-                let name = attr.path.segments.last().unwrap().ident.to_string();
+                let name = attr.path().segments.last().unwrap().ident.to_string();
                 let needs_clone =
                     name == "primary_span" && matches!(inner_ty, FieldInnerTy::Vec(_));
                 let (binding, needs_destructure) = if needs_clone {
@@ -343,11 +313,10 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
         binding: TokenStream,
     ) -> Result<TokenStream, DiagnosticDeriveError> {
         let diag = &self.parent.diag;
-        let meta = attr.parse_meta()?;
 
-        let ident = &attr.path.segments.last().unwrap().ident;
+        let ident = &attr.path().segments.last().unwrap().ident;
         let name = ident.to_string();
-        match (&meta, name.as_str()) {
+        match (&attr.meta, name.as_str()) {
             // Don't need to do anything - by virtue of the attribute existing, the
             // `set_arg` call will not be generated.
             (Meta::Path(_), "skip_arg") => return Ok(quote! {}),
@@ -361,7 +330,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                         });
                     }
                     DiagnosticDeriveKind::LintDiagnostic => {
-                        throw_invalid_attr!(attr, &meta, |diag| {
+                        throw_invalid_attr!(attr, |diag| {
                             diag.help("the `primary_span` field attribute is not valid for lint diagnostics")
                         })
                     }
@@ -378,26 +347,34 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                     return Ok(quote! { #diag.subdiagnostic(#binding); });
                 }
             }
-            (Meta::List(MetaList { ref nested, .. }), "subdiagnostic") => {
-                if nested.len() == 1
-                    && let Some(NestedMeta::Meta(Meta::Path(path))) = nested.first()
-                    && path.is_ident("eager") {
-                        let handler = match &self.parent.kind {
-                            DiagnosticDeriveKind::Diagnostic { handler } => handler,
-                            DiagnosticDeriveKind::LintDiagnostic => {
-                                throw_invalid_attr!(attr, &meta, |diag| {
-                                    diag.help("eager subdiagnostics are not supported on lints")
-                                })
-                            }
-                        };
-                        return Ok(quote! { #diag.eager_subdiagnostic(#handler, #binding); });
-                } else {
-                    throw_invalid_attr!(attr, &meta, |diag| {
-                        diag.help(
-                            "`eager` is the only supported nested attribute for `subdiagnostic`",
-                        )
-                    })
+            (Meta::List(meta_list), "subdiagnostic") => {
+                let err = || {
+                    span_err(
+                        meta_list.span().unwrap(),
+                        "`eager` is the only supported nested attribute for `subdiagnostic`",
+                    )
+                    .emit();
+                };
+
+                let Ok(p): Result<Path, _> = meta_list.parse_args() else {
+                    err();
+                    return Ok(quote! {});
+                };
+
+                if !p.is_ident("eager") {
+                    err();
+                    return Ok(quote! {});
                 }
+
+                let handler = match &self.parent.kind {
+                    DiagnosticDeriveKind::Diagnostic { handler } => handler,
+                    DiagnosticDeriveKind::LintDiagnostic => {
+                        throw_invalid_attr!(attr, |diag| {
+                            diag.help("eager subdiagnostics are not supported on lints")
+                        })
+                    }
+                };
+                return Ok(quote! { #diag.eager_subdiagnostic(#handler, #binding); });
             }
             _ => (),
         }
@@ -414,12 +391,17 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                 Ok(self.add_spanned_subdiagnostic(binding, &fn_ident, slug))
             }
             SubdiagnosticKind::Note | SubdiagnosticKind::Help | SubdiagnosticKind::Warn => {
-                if type_matches_path(info.ty.inner_type(), &["rustc_span", "Span"]) {
+                let inner = info.ty.inner_type();
+                if type_matches_path(inner, &["rustc_span", "Span"])
+                    || type_matches_path(inner, &["rustc_span", "MultiSpan"])
+                {
                     Ok(self.add_spanned_subdiagnostic(binding, &fn_ident, slug))
-                } else if type_is_unit(info.ty.inner_type()) {
+                } else if type_is_unit(inner)
+                    || (matches!(info.ty, FieldInnerTy::Plain(_)) && type_is_bool(inner))
+                {
                     Ok(self.add_subdiagnostic(&fn_ident, slug))
                 } else {
-                    report_type_error(attr, "`Span` or `()`")?
+                    report_type_error(attr, "`Span`, `MultiSpan`, `bool` or `()`")?
                 }
             }
             SubdiagnosticKind::Suggestion {
@@ -429,7 +411,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                 code_init,
             } => {
                 if let FieldInnerTy::Vec(_) = info.ty {
-                    throw_invalid_attr!(attr, &meta, |diag| {
+                    throw_invalid_attr!(attr, |diag| {
                         diag
                         .note("`#[suggestion(...)]` applied to `Vec` field is ambiguous")
                         .help("to show a suggestion consisting of multiple parts, use a `Subdiagnostic` annotated with `#[multipart_suggestion(...)]`")
diff --git a/compiler/rustc_macros/src/diagnostics/error.rs b/compiler/rustc_macros/src/diagnostics/error.rs
index 2d62d5931..b37dc826d 100644
--- a/compiler/rustc_macros/src/diagnostics/error.rs
+++ b/compiler/rustc_macros/src/diagnostics/error.rs
@@ -1,7 +1,7 @@
 use proc_macro::{Diagnostic, Level, MultiSpan};
 use proc_macro2::TokenStream;
 use quote::quote;
-use syn::{spanned::Spanned, Attribute, Error as SynError, Meta, NestedMeta};
+use syn::{spanned::Spanned, Attribute, Error as SynError, Meta};
 
 #[derive(Debug)]
 pub(crate) enum DiagnosticDeriveError {
@@ -53,6 +53,7 @@ fn path_to_string(path: &syn::Path) -> String {
 }
 
 /// Returns an error diagnostic on span `span` with msg `msg`.
+#[must_use]
 pub(crate) fn span_err(span: impl MultiSpan, msg: &str) -> Diagnostic {
     Diagnostic::spanned(span, Level::Error, msg)
 }
@@ -72,10 +73,10 @@ macro_rules! throw_span_err {
 pub(crate) use throw_span_err;
 
 /// Returns an error diagnostic for an invalid attribute.
-pub(crate) fn invalid_attr(attr: &Attribute, meta: &Meta) -> Diagnostic {
+pub(crate) fn invalid_attr(attr: &Attribute) -> Diagnostic {
     let span = attr.span().unwrap();
-    let path = path_to_string(&attr.path);
-    match meta {
+    let path = path_to_string(attr.path());
+    match attr.meta {
         Meta::Path(_) => span_err(span, &format!("`#[{path}]` is not a valid attribute")),
         Meta::NameValue(_) => {
             span_err(span, &format!("`#[{path} = ...]` is not a valid attribute"))
@@ -89,51 +90,11 @@ pub(crate) fn invalid_attr(attr: &Attribute, meta: &Meta) -> Diagnostic {
 ///
 /// For methods that return a `Result<_, DiagnosticDeriveError>`:
 macro_rules! throw_invalid_attr {
-    ($attr:expr, $meta:expr) => {{ throw_invalid_attr!($attr, $meta, |diag| diag) }};
-    ($attr:expr, $meta:expr, $f:expr) => {{
-        let diag = crate::diagnostics::error::invalid_attr($attr, $meta);
+    ($attr:expr) => {{ throw_invalid_attr!($attr, |diag| diag) }};
+    ($attr:expr, $f:expr) => {{
+        let diag = crate::diagnostics::error::invalid_attr($attr);
         return Err(crate::diagnostics::error::_throw_err(diag, $f));
     }};
 }
 
 pub(crate) use throw_invalid_attr;
-
-/// Returns an error diagnostic for an invalid nested attribute.
-pub(crate) fn invalid_nested_attr(attr: &Attribute, nested: &NestedMeta) -> Diagnostic {
-    let name = attr.path.segments.last().unwrap().ident.to_string();
-    let name = name.as_str();
-
-    let span = nested.span().unwrap();
-    let meta = match nested {
-        syn::NestedMeta::Meta(meta) => meta,
-        syn::NestedMeta::Lit(_) => {
-            return span_err(span, &format!("`#[{name}(\"...\")]` is not a valid attribute"));
-        }
-    };
-
-    let span = meta.span().unwrap();
-    let path = path_to_string(meta.path());
-    match meta {
-        Meta::NameValue(..) => {
-            span_err(span, &format!("`#[{name}({path} = ...)]` is not a valid attribute"))
-        }
-        Meta::Path(..) => span_err(span, &format!("`#[{name}({path})]` is not a valid attribute")),
-        Meta::List(..) => {
-            span_err(span, &format!("`#[{name}({path}(...))]` is not a valid attribute"))
-        }
-    }
-}
-
-/// Emit an error diagnostic for an invalid nested attribute (optionally performing additional
-/// decoration using the `FnOnce` passed in `diag`) and return `Err(ErrorHandled)`.
-///
-/// For methods that return a `Result<_, DiagnosticDeriveError>`:
-macro_rules! throw_invalid_nested_attr {
-    ($attr:expr, $nested_attr:expr) => {{ throw_invalid_nested_attr!($attr, $nested_attr, |diag| diag) }};
-    ($attr:expr, $nested_attr:expr, $f:expr) => {{
-        let diag = crate::diagnostics::error::invalid_nested_attr($attr, $nested_attr);
-        return Err(crate::diagnostics::error::_throw_err(diag, $f));
-    }};
-}
-
-pub(crate) use throw_invalid_nested_attr;
diff --git a/compiler/rustc_macros/src/diagnostics/fluent.rs b/compiler/rustc_macros/src/diagnostics/fluent.rs
index 38c0f4895..607d51f56 100644
--- a/compiler/rustc_macros/src/diagnostics/fluent.rs
+++ b/compiler/rustc_macros/src/diagnostics/fluent.rs
@@ -15,8 +15,7 @@ use proc_macro2::TokenStream;
 use quote::quote;
 use std::{
     collections::{HashMap, HashSet},
-    fs::File,
-    io::Read,
+    fs::read_to_string,
     path::{Path, PathBuf},
 };
 use syn::{parse_macro_input, Ident, LitStr};
@@ -95,20 +94,28 @@ pub(crate) fn fluent_messages(input: proc_macro::TokenStream) -> proc_macro::Tok
 
     // As this macro also outputs an `include_str!` for this file, the macro will always be
     // re-executed when the file changes.
-    let mut resource_file = match File::open(absolute_ftl_path) {
-        Ok(resource_file) => resource_file,
+    let resource_contents = match read_to_string(absolute_ftl_path) {
+        Ok(resource_contents) => resource_contents,
         Err(e) => {
-            Diagnostic::spanned(resource_span, Level::Error, "could not open Fluent resource")
-                .note(e.to_string())
-                .emit();
+            Diagnostic::spanned(
+                resource_span,
+                Level::Error,
+                format!("could not open Fluent resource: {e}"),
+            )
+            .emit();
             return failed(&crate_name);
         }
     };
-    let mut resource_contents = String::new();
-    if let Err(e) = resource_file.read_to_string(&mut resource_contents) {
-        Diagnostic::spanned(resource_span, Level::Error, "could not read Fluent resource")
-            .note(e.to_string())
-            .emit();
+    let mut bad = false;
+    for esc in ["\\n", "\\\"", "\\'"] {
+        for _ in resource_contents.matches(esc) {
+            bad = true;
+            Diagnostic::spanned(resource_span, Level::Error, format!("invalid escape `{esc}` in Fluent resource"))
+                .note("Fluent does not interpret these escape sequences (<https://projectfluent.org/fluent/guide/special.html>)")
+                .emit();
+        }
+    }
+    if bad {
         return failed(&crate_name);
     }
 
diff --git a/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs b/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
index 90660fc1f..62d49c1c6 100644
--- a/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
+++ b/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
@@ -1,8 +1,7 @@
 #![deny(unused_must_use)]
 
 use crate::diagnostics::error::{
-    invalid_attr, span_err, throw_invalid_attr, throw_invalid_nested_attr, throw_span_err,
-    DiagnosticDeriveError,
+    invalid_attr, span_err, throw_invalid_attr, throw_span_err, DiagnosticDeriveError,
 };
 use crate::diagnostics::utils::{
     build_field_mapping, is_doc_comment, new_code_ident,
@@ -11,7 +10,7 @@ use crate::diagnostics::utils::{
 };
 use proc_macro2::TokenStream;
 use quote::{format_ident, quote};
-use syn::{spanned::Spanned, Attribute, Meta, MetaList, NestedMeta, Path};
+use syn::{spanned::Spanned, Attribute, Meta, MetaList, Path};
 use synstructure::{BindingInfo, Structure, VariantInfo};
 
 use super::utils::{build_suggestion_code, AllowMultipleAlternatives};
@@ -39,7 +38,8 @@ impl SubdiagnosticDeriveBuilder {
                     span_err(
                         span,
                         "`#[derive(Subdiagnostic)]` can only be used on structs and enums",
-                    );
+                    )
+                    .emit();
                 }
             }
 
@@ -192,7 +192,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
             };
 
             let Some(slug) = slug else {
-                let name = attr.path.segments.last().unwrap().ident.to_string();
+                let name = attr.path().segments.last().unwrap().ident.to_string();
                 let name = name.as_str();
 
                 throw_span_err!(
@@ -265,17 +265,18 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
         info: FieldInfo<'_>,
         clone_suggestion_code: bool,
     ) -> Result<TokenStream, DiagnosticDeriveError> {
-        let meta = attr.parse_meta()?;
-        match meta {
-            Meta::Path(path) => self.generate_field_code_inner_path(kind_stats, attr, info, path),
-            Meta::List(list @ MetaList { .. }) => self.generate_field_code_inner_list(
+        match &attr.meta {
+            Meta::Path(path) => {
+                self.generate_field_code_inner_path(kind_stats, attr, info, path.clone())
+            }
+            Meta::List(list) => self.generate_field_code_inner_list(
                 kind_stats,
                 attr,
                 info,
                 list,
                 clone_suggestion_code,
             ),
-            _ => throw_invalid_attr!(attr, &meta),
+            _ => throw_invalid_attr!(attr),
         }
     }
 
@@ -296,7 +297,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
             "skip_arg" => Ok(quote! {}),
             "primary_span" => {
                 if kind_stats.has_multipart_suggestion {
-                    invalid_attr(attr, &Meta::Path(path))
+                    invalid_attr(attr)
                         .help(
                             "multipart suggestions use one or more `#[suggestion_part]`s rather \
                             than one `#[primary_span]`",
@@ -309,7 +310,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
                     // FIXME(#100717): support `Option<Span>` on `primary_span` like in the
                     // diagnostic derive
                     if !matches!(info.ty, FieldInnerTy::Plain(_)) {
-                        throw_invalid_attr!(attr, &Meta::Path(path), |diag| {
+                        throw_invalid_attr!(attr, |diag| {
                             let diag = diag.note("there must be exactly one primary span");
 
                             if kind_stats.has_normal_suggestion {
@@ -335,7 +336,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
                     span_err(span, "`#[suggestion_part(...)]` attribute without `code = \"...\"`")
                         .emit();
                 } else {
-                    invalid_attr(attr, &Meta::Path(path))
+                    invalid_attr(attr)
                         .help(
                             "`#[suggestion_part(...)]` is only valid in multipart suggestions, \
                              use `#[primary_span]` instead",
@@ -375,7 +376,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
                     span_attrs.push("primary_span")
                 }
 
-                invalid_attr(attr, &Meta::Path(path))
+                invalid_attr(attr)
                     .help(format!(
                         "only `{}`, `applicability` and `skip_arg` are valid field attributes",
                         span_attrs.join(", ")
@@ -394,7 +395,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
         kind_stats: KindsStatistics,
         attr: &Attribute,
         info: FieldInfo<'_>,
-        list: MetaList,
+        list: &MetaList,
         clone_suggestion_code: bool,
     ) -> Result<TokenStream, DiagnosticDeriveError> {
         let span = attr.span().unwrap();
@@ -405,7 +406,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
         match name {
             "suggestion_part" => {
                 if !kind_stats.has_multipart_suggestion {
-                    throw_invalid_attr!(attr, &Meta::List(list), |diag| {
+                    throw_invalid_attr!(attr, |diag| {
                         diag.help(
                             "`#[suggestion_part(...)]` is only valid in multipart suggestions",
                         )
@@ -417,31 +418,27 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
                 report_error_if_not_applied_to_span(attr, &info)?;
 
                 let mut code = None;
-                for nested_attr in list.nested.iter() {
-                    let NestedMeta::Meta(ref meta) = nested_attr else {
-                        throw_invalid_nested_attr!(attr, nested_attr);
-                    };
-
-                    let span = meta.span().unwrap();
-                    let nested_name = meta.path().segments.last().unwrap().ident.to_string();
-                    let nested_name = nested_name.as_str();
-
-                    match nested_name {
-                        "code" => {
-                            let code_field = new_code_ident();
-                            let formatting_init = build_suggestion_code(
-                                &code_field,
-                                meta,
-                                self,
-                                AllowMultipleAlternatives::No,
-                            );
-                            code.set_once((code_field, formatting_init), span);
-                        }
-                        _ => throw_invalid_nested_attr!(attr, nested_attr, |diag| {
-                            diag.help("`code` is the only valid nested attribute")
-                        }),
+
+                list.parse_nested_meta(|nested| {
+                    if nested.path.is_ident("code") {
+                        let code_field = new_code_ident();
+                        let span = nested.path.span().unwrap();
+                        let formatting_init = build_suggestion_code(
+                            &code_field,
+                            nested,
+                            self,
+                            AllowMultipleAlternatives::No,
+                        );
+                        code.set_once((code_field, formatting_init), span);
+                    } else {
+                        span_err(
+                            nested.path.span().unwrap(),
+                            "`code` is the only valid nested attribute",
+                        )
+                        .emit();
                     }
-                }
+                    Ok(())
+                })?;
 
                 let Some((code_field, formatting_init)) = code.value() else {
                     span_err(span, "`#[suggestion_part(...)]` attribute without `code = \"...\"`")
@@ -458,7 +455,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
                 };
                 Ok(quote! { suggestions.push((#binding, #code_field)); })
             }
-            _ => throw_invalid_attr!(attr, &Meta::List(list), |diag| {
+            _ => throw_invalid_attr!(attr, |diag| {
                 let mut span_attrs = vec![];
                 if kind_stats.has_multipart_suggestion {
                     span_attrs.push("suggestion_part");
diff --git a/compiler/rustc_macros/src/diagnostics/utils.rs b/compiler/rustc_macros/src/diagnostics/utils.rs
index 27b8f676f..b9b09c662 100644
--- a/compiler/rustc_macros/src/diagnostics/utils.rs
+++ b/compiler/rustc_macros/src/diagnostics/utils.rs
@@ -1,5 +1,5 @@
 use crate::diagnostics::error::{
-    span_err, throw_invalid_attr, throw_invalid_nested_attr, throw_span_err, DiagnosticDeriveError,
+    span_err, throw_invalid_attr, throw_span_err, DiagnosticDeriveError,
 };
 use proc_macro::Span;
 use proc_macro2::{Ident, TokenStream};
@@ -8,11 +8,13 @@ use std::cell::RefCell;
 use std::collections::{BTreeSet, HashMap};
 use std::fmt;
 use std::str::FromStr;
+use syn::meta::ParseNestedMeta;
+use syn::punctuated::Punctuated;
+use syn::{parenthesized, LitStr, Path, Token};
 use syn::{spanned::Spanned, Attribute, Field, Meta, Type, TypeTuple};
-use syn::{MetaList, MetaNameValue, NestedMeta, Path};
 use synstructure::{BindingInfo, VariantInfo};
 
-use super::error::{invalid_attr, invalid_nested_attr};
+use super::error::invalid_attr;
 
 thread_local! {
     pub static CODE_IDENT_COUNT: RefCell<u32> = RefCell::new(0);
@@ -50,13 +52,18 @@ pub(crate) fn type_is_unit(ty: &Type) -> bool {
     if let Type::Tuple(TypeTuple { elems, .. }) = ty { elems.is_empty() } else { false }
 }
 
+/// Checks whether the type `ty` is `bool`.
+pub(crate) fn type_is_bool(ty: &Type) -> bool {
+    type_matches_path(ty, &["bool"])
+}
+
 /// Reports a type error for field with `attr`.
 pub(crate) fn report_type_error(
     attr: &Attribute,
     ty_name: &str,
 ) -> Result<!, DiagnosticDeriveError> {
-    let name = attr.path.segments.last().unwrap().ident.to_string();
-    let meta = attr.parse_meta()?;
+    let name = attr.path().segments.last().unwrap().ident.to_string();
+    let meta = &attr.meta;
 
     throw_span_err!(
         attr.span().unwrap(),
@@ -192,6 +199,11 @@ impl<'ty> FieldInnerTy<'ty> {
                     #inner
                 }
             },
+            FieldInnerTy::Plain(t) if type_is_bool(t) => quote! {
+                if #binding {
+                    #inner
+                }
+            },
             FieldInnerTy::Plain(..) => quote! { #inner },
         }
     }
@@ -408,59 +420,62 @@ pub(super) enum AllowMultipleAlternatives {
     Yes,
 }
 
+fn parse_suggestion_values(
+    nested: ParseNestedMeta<'_>,
+    allow_multiple: AllowMultipleAlternatives,
+) -> syn::Result<Vec<LitStr>> {
+    let values = if let Ok(val) = nested.value() {
+        vec![val.parse()?]
+    } else {
+        let content;
+        parenthesized!(content in nested.input);
+
+        if let AllowMultipleAlternatives::No = allow_multiple {
+            span_err(
+                nested.input.span().unwrap(),
+                "expected exactly one string literal for `code = ...`",
+            )
+            .emit();
+            vec![]
+        } else {
+            let literals = Punctuated::<LitStr, Token![,]>::parse_terminated(&content);
+
+            match literals {
+                Ok(p) if p.is_empty() => {
+                    span_err(
+                        content.span().unwrap(),
+                        "expected at least one string literal for `code(...)`",
+                    )
+                    .emit();
+                    vec![]
+                }
+                Ok(p) => p.into_iter().collect(),
+                Err(_) => {
+                    span_err(
+                        content.span().unwrap(),
+                        "`code(...)` must contain only string literals",
+                    )
+                    .emit();
+                    vec![]
+                }
+            }
+        }
+    };
+
+    Ok(values)
+}
+
 /// Constructs the `format!()` invocation(s) necessary for a `#[suggestion*(code = "foo")]` or
 /// `#[suggestion*(code("foo", "bar"))]` attribute field
 pub(super) fn build_suggestion_code(
     code_field: &Ident,
-    meta: &Meta,
+    nested: ParseNestedMeta<'_>,
     fields: &impl HasFieldMap,
     allow_multiple: AllowMultipleAlternatives,
 ) -> TokenStream {
-    let values = match meta {
-        // `code = "foo"`
-        Meta::NameValue(MetaNameValue { lit: syn::Lit::Str(s), .. }) => vec![s],
-        // `code("foo", "bar")`
-        Meta::List(MetaList { nested, .. }) => {
-            if let AllowMultipleAlternatives::No = allow_multiple {
-                span_err(
-                    meta.span().unwrap(),
-                    "expected exactly one string literal for `code = ...`",
-                )
-                .emit();
-                vec![]
-            } else if nested.is_empty() {
-                span_err(
-                    meta.span().unwrap(),
-                    "expected at least one string literal for `code(...)`",
-                )
-                .emit();
-                vec![]
-            } else {
-                nested
-                    .into_iter()
-                    .filter_map(|item| {
-                        if let NestedMeta::Lit(syn::Lit::Str(s)) = item {
-                            Some(s)
-                        } else {
-                            span_err(
-                                item.span().unwrap(),
-                                "`code(...)` must contain only string literals",
-                            )
-                            .emit();
-                            None
-                        }
-                    })
-                    .collect()
-            }
-        }
-        _ => {
-            span_err(
-                meta.span().unwrap(),
-                r#"`code = "..."`/`code(...)` must contain only string literals"#,
-            )
-            .emit();
-            vec![]
-        }
+    let values = match parse_suggestion_values(nested, allow_multiple) {
+        Ok(x) => x,
+        Err(e) => return e.into_compile_error(),
     };
 
     if let AllowMultipleAlternatives::Yes = allow_multiple {
@@ -591,11 +606,9 @@ impl SubdiagnosticKind {
 
         let span = attr.span().unwrap();
 
-        let name = attr.path.segments.last().unwrap().ident.to_string();
+        let name = attr.path().segments.last().unwrap().ident.to_string();
         let name = name.as_str();
 
-        let meta = attr.parse_meta()?;
-
         let mut kind = match name {
             "label" => SubdiagnosticKind::Label,
             "note" => SubdiagnosticKind::Note,
@@ -608,7 +621,7 @@ impl SubdiagnosticKind {
                     name.strip_prefix("suggestion").and_then(SuggestionKind::from_suffix)
                 {
                     if suggestion_kind != SuggestionKind::Normal {
-                        invalid_attr(attr, &meta)
+                        invalid_attr(attr)
                             .help(format!(
                                 r#"Use `#[suggestion(..., style = "{suggestion_kind}")]` instead"#
                             ))
@@ -625,7 +638,7 @@ impl SubdiagnosticKind {
                     name.strip_prefix("multipart_suggestion").and_then(SuggestionKind::from_suffix)
                 {
                     if suggestion_kind != SuggestionKind::Normal {
-                        invalid_attr(attr, &meta)
+                        invalid_attr(attr)
                             .help(format!(
                                 r#"Use `#[multipart_suggestion(..., style = "{suggestion_kind}")]` instead"#
                             ))
@@ -637,16 +650,16 @@ impl SubdiagnosticKind {
                         applicability: None,
                     }
                 } else {
-                    throw_invalid_attr!(attr, &meta);
+                    throw_invalid_attr!(attr);
                 }
             }
         };
 
-        let nested = match meta {
-            Meta::List(MetaList { ref nested, .. }) => {
+        let list = match &attr.meta {
+            Meta::List(list) => {
                 // An attribute with properties, such as `#[suggestion(code = "...")]` or
                 // `#[error(some::slug)]`
-                nested
+                list
             }
             Meta::Path(_) => {
                 // An attribute without a slug or other properties, such as `#[note]` - return
@@ -668,69 +681,68 @@ impl SubdiagnosticKind {
                 }
             }
             _ => {
-                throw_invalid_attr!(attr, &meta)
+                throw_invalid_attr!(attr)
             }
         };
 
         let mut code = None;
         let mut suggestion_kind = None;
 
-        let mut nested_iter = nested.into_iter().peekable();
+        let mut first = true;
+        let mut slug = None;
 
-        // Peek at the first nested attribute: if it's a slug path, consume it.
-        let slug = if let Some(NestedMeta::Meta(Meta::Path(path))) = nested_iter.peek() {
-            let path = path.clone();
-            // Advance the iterator.
-            nested_iter.next();
-            Some(path)
-        } else {
-            None
-        };
-
-        for nested_attr in nested_iter {
-            let meta = match nested_attr {
-                NestedMeta::Meta(ref meta) => meta,
-                NestedMeta::Lit(_) => {
-                    invalid_nested_attr(attr, nested_attr).emit();
-                    continue;
+        list.parse_nested_meta(|nested| {
+            if nested.input.is_empty() || nested.input.peek(Token![,]) {
+                if first {
+                    slug = Some(nested.path);
+                } else {
+                    span_err(nested.input.span().unwrap(), "a diagnostic slug must be the first argument to the attribute").emit();
                 }
-            };
 
-            let span = meta.span().unwrap();
-            let nested_name = meta.path().segments.last().unwrap().ident.to_string();
+                first = false;
+                return Ok(());
+            }
+
+            first = false;
+
+            let nested_name = nested.path.segments.last().unwrap().ident.to_string();
             let nested_name = nested_name.as_str();
 
-            let string_value = match meta {
-                Meta::NameValue(MetaNameValue { lit: syn::Lit::Str(value), .. }) => Some(value),
+            let path_span = nested.path.span().unwrap();
+            let val_span = nested.input.span().unwrap();
 
-                Meta::Path(_) => throw_invalid_nested_attr!(attr, nested_attr, |diag| {
-                    diag.help("a diagnostic slug must be the first argument to the attribute")
-                }),
-                _ => None,
-            };
+            macro_rules! get_string {
+                () => {{
+                    let Ok(value) = nested.value().and_then(|x| x.parse::<LitStr>()) else {
+                        span_err(val_span, "expected `= \"xxx\"`").emit();
+                        return Ok(());
+                    };
+                    value
+                }};
+            }
+
+            let mut has_errors = false;
+            let input = nested.input;
 
             match (nested_name, &mut kind) {
                 ("code", SubdiagnosticKind::Suggestion { code_field, .. }) => {
                     let code_init = build_suggestion_code(
                         code_field,
-                        meta,
+                        nested,
                         fields,
                         AllowMultipleAlternatives::Yes,
                     );
-                    code.set_once(code_init, span);
+                    code.set_once(code_init, path_span);
                 }
                 (
                     "applicability",
                     SubdiagnosticKind::Suggestion { ref mut applicability, .. }
                     | SubdiagnosticKind::MultipartSuggestion { ref mut applicability, .. },
                 ) => {
-                    let Some(value) = string_value else {
-                        invalid_nested_attr(attr, nested_attr).emit();
-                        continue;
-                    };
-
+                    let value = get_string!();
                     let value = Applicability::from_str(&value.value()).unwrap_or_else(|()| {
-                        span_err(span, "invalid applicability").emit();
+                        span_err(value.span().unwrap(), "invalid applicability").emit();
+                        has_errors = true;
                         Applicability::Unspecified
                     });
                     applicability.set_once(value, span);
@@ -740,15 +752,13 @@ impl SubdiagnosticKind {
                     SubdiagnosticKind::Suggestion { .. }
                     | SubdiagnosticKind::MultipartSuggestion { .. },
                 ) => {
-                    let Some(value) = string_value else {
-                        invalid_nested_attr(attr, nested_attr).emit();
-                        continue;
-                    };
+                    let value = get_string!();
 
                     let value = value.value().parse().unwrap_or_else(|()| {
                         span_err(value.span().unwrap(), "invalid suggestion style")
                             .help("valid styles are `normal`, `short`, `hidden`, `verbose` and `tool-only`")
                             .emit();
+                        has_errors = true;
                         SuggestionKind::Normal
                     });
 
@@ -757,22 +767,32 @@ impl SubdiagnosticKind {
 
                 // Invalid nested attribute
                 (_, SubdiagnosticKind::Suggestion { .. }) => {
-                    invalid_nested_attr(attr, nested_attr)
+                    span_err(path_span, "invalid nested attribute")
                         .help(
                             "only `style`, `code` and `applicability` are valid nested attributes",
                         )
                         .emit();
+                    has_errors = true;
                 }
                 (_, SubdiagnosticKind::MultipartSuggestion { .. }) => {
-                    invalid_nested_attr(attr, nested_attr)
+                    span_err(path_span, "invalid nested attribute")
                         .help("only `style` and `applicability` are valid nested attributes")
-                        .emit()
+                        .emit();
+                    has_errors = true;
                 }
                 _ => {
-                    invalid_nested_attr(attr, nested_attr).emit();
+                    span_err(path_span, "invalid nested attribute").emit();
+                    has_errors = true;
                 }
             }
-        }
+
+            if has_errors {
+                // Consume the rest of the input to avoid spamming errors
+                let _ = input.parse::<TokenStream>();
+            }
+
+            Ok(())
+        })?;
 
         match kind {
             SubdiagnosticKind::Suggestion {
@@ -835,5 +855,5 @@ pub(super) fn should_generate_set_arg(field: &Field) -> bool {
 }
 
 pub(super) fn is_doc_comment(attr: &Attribute) -> bool {
-    attr.path.segments.last().unwrap().ident == "doc"
+    attr.path().segments.last().unwrap().ident == "doc"
 }
diff --git a/compiler/rustc_macros/src/hash_stable.rs b/compiler/rustc_macros/src/hash_stable.rs
index 63bdcea87..75a2f7009 100644
--- a/compiler/rustc_macros/src/hash_stable.rs
+++ b/compiler/rustc_macros/src/hash_stable.rs
@@ -1,6 +1,6 @@
 use proc_macro2::{self, Ident};
 use quote::quote;
-use syn::{self, parse_quote, Meta, NestedMeta};
+use syn::{self, parse_quote};
 
 struct Attributes {
     ignore: bool,
@@ -10,32 +10,29 @@ struct Attributes {
 fn parse_attributes(field: &syn::Field) -> Attributes {
     let mut attrs = Attributes { ignore: false, project: None };
     for attr in &field.attrs {
-        if let Ok(meta) = attr.parse_meta() {
-            if !meta.path().is_ident("stable_hasher") {
-                continue;
+        let meta = &attr.meta;
+        if !meta.path().is_ident("stable_hasher") {
+            continue;
+        }
+        let mut any_attr = false;
+        let _ = attr.parse_nested_meta(|nested| {
+            if nested.path.is_ident("ignore") {
+                attrs.ignore = true;
+                any_attr = true;
             }
-            let mut any_attr = false;
-            if let Meta::List(list) = meta {
-                for nested in list.nested.iter() {
-                    if let NestedMeta::Meta(meta) = nested {
-                        if meta.path().is_ident("ignore") {
-                            attrs.ignore = true;
-                            any_attr = true;
-                        }
-                        if meta.path().is_ident("project") {
-                            if let Meta::List(list) = meta {
-                                if let Some(NestedMeta::Meta(meta)) = list.nested.iter().next() {
-                                    attrs.project = meta.path().get_ident().cloned();
-                                    any_attr = true;
-                                }
-                            }
-                        }
+            if nested.path.is_ident("project") {
+                let _ = nested.parse_nested_meta(|meta| {
+                    if attrs.project.is_none() {
+                        attrs.project = meta.path.get_ident().cloned();
                     }
-                }
-            }
-            if !any_attr {
-                panic!("error parsing stable_hasher");
+                    any_attr = true;
+                    Ok(())
+                });
             }
+            Ok(())
+        });
+        if !any_attr {
+            panic!("error parsing stable_hasher");
         }
     }
     attrs
diff --git a/compiler/rustc_macros/src/newtype.rs b/compiler/rustc_macros/src/newtype.rs
index 89ea89cf5..78a6f7488 100644
--- a/compiler/rustc_macros/src/newtype.rs
+++ b/compiler/rustc_macros/src/newtype.rs
@@ -25,7 +25,7 @@ impl Parse for Newtype {
         let mut encodable = true;
         let mut ord = true;
 
-        attrs.retain(|attr| match attr.path.get_ident() {
+        attrs.retain(|attr| match attr.path().get_ident() {
             Some(ident) => match &*ident.to_string() {
                 "custom_encodable" => {
                     encodable = false;
@@ -36,22 +36,22 @@ impl Parse for Newtype {
                     false
                 }
                 "max" => {
-                    let Ok(Meta::NameValue(literal) )= attr.parse_meta() else {
+                    let Meta::NameValue(MetaNameValue { value: Expr::Lit(lit), .. }) = &attr.meta else {
                         panic!("#[max = NUMBER] attribute requires max value");
                     };
 
-                    if let Some(old) = max.replace(literal.lit) {
+                    if let Some(old) = max.replace(lit.lit.clone()) {
                         panic!("Specified multiple max: {old:?}");
                     }
 
                     false
                 }
                 "debug_format" => {
-                    let Ok(Meta::NameValue(literal) )= attr.parse_meta() else {
+                    let Meta::NameValue(MetaNameValue { value: Expr::Lit(lit), .. }) = &attr.meta else {
                         panic!("#[debug_format = FMT] attribute requires a format");
                     };
 
-                    if let Some(old) = debug_format.replace(literal.lit) {
+                    if let Some(old) = debug_format.replace(lit.lit.clone()) {
                         panic!("Specified multiple debug format options: {old:?}");
                     }
 
diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs
index 08e42a8a0..f85ba3800 100644
--- a/compiler/rustc_macros/src/query.rs
+++ b/compiler/rustc_macros/src/query.rs
@@ -15,7 +15,7 @@ mod kw {
 /// Ensures only doc comment attributes are used
 fn check_attributes(attrs: Vec<Attribute>) -> Result<Vec<Attribute>> {
     let inner = |attr: Attribute| {
-        if !attr.path.is_ident("doc") {
+        if !attr.path().is_ident("doc") {
             Err(Error::new(attr.span(), "attributes not supported on queries"))
         } else if attr.style != AttrStyle::Outer {
             Err(Error::new(
@@ -48,7 +48,7 @@ impl Parse for Query {
         let name: Ident = input.parse()?;
         let arg_content;
         parenthesized!(arg_content in input);
-        let key = arg_content.parse()?;
+        let key = Pat::parse_single(&arg_content)?;
         arg_content.parse::<Token![:]>()?;
         let arg = arg_content.parse()?;
         let result = input.parse()?;
@@ -158,7 +158,7 @@ fn parse_query_modifiers(input: ParseStream<'_>) -> Result<QueryModifiers> {
             } else {
                 None
             };
-            let list = attr_content.parse_terminated(Expr::parse)?;
+            let list = attr_content.parse_terminated(Expr::parse, Token![,])?;
             try_insert!(desc = (tcx, list));
         } else if modifier == "cache_on_disk_if" {
             // Parse a cache modifier like:
@@ -166,7 +166,7 @@ fn parse_query_modifiers(input: ParseStream<'_>) -> Result<QueryModifiers> {
             let args = if input.peek(token::Paren) {
                 let args;
                 parenthesized!(args in input);
-                let tcx = args.parse()?;
+                let tcx = Pat::parse_single(&args)?;
                 Some(tcx)
             } else {
                 None
diff --git a/compiler/rustc_macros/src/type_foldable.rs b/compiler/rustc_macros/src/type_foldable.rs
index 388e254cd..5ee4d8793 100644
--- a/compiler/rustc_macros/src/type_foldable.rs
+++ b/compiler/rustc_macros/src/type_foldable.rs
@@ -1,5 +1,5 @@
 use quote::{quote, ToTokens};
-use syn::{parse_quote, Attribute, Meta, NestedMeta};
+use syn::parse_quote;
 
 pub fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
     if let syn::Data::Union(_) = s.ast().data {
@@ -17,21 +17,20 @@ pub fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::
         vi.construct(|_, index| {
             let bind = &bindings[index];
 
+            let mut fixed = false;
+
             // retain value of fields with #[type_foldable(identity)]
-            let fixed = bind
-                .ast()
-                .attrs
-                .iter()
-                .map(Attribute::parse_meta)
-                .filter_map(Result::ok)
-                .flat_map(|attr| match attr {
-                    Meta::List(list) if list.path.is_ident("type_foldable") => list.nested,
-                    _ => Default::default(),
-                })
-                .any(|nested| match nested {
-                    NestedMeta::Meta(Meta::Path(path)) => path.is_ident("identity"),
-                    _ => false,
+            bind.ast().attrs.iter().for_each(|x| {
+                if !x.path().is_ident("type_foldable") {
+                    return;
+                }
+                let _ = x.parse_nested_meta(|nested| {
+                    if nested.path.is_ident("identity") {
+                        fixed = true;
+                    }
+                    Ok(())
                 });
+            });
 
             if fixed {
                 bind.to_token_stream()
diff --git a/compiler/rustc_macros/src/type_visitable.rs b/compiler/rustc_macros/src/type_visitable.rs
index f6f4c4779..dcd505a10 100644
--- a/compiler/rustc_macros/src/type_visitable.rs
+++ b/compiler/rustc_macros/src/type_visitable.rs
@@ -1,5 +1,5 @@
 use quote::quote;
-use syn::{parse_quote, Attribute, Meta, NestedMeta};
+use syn::parse_quote;
 
 pub fn type_visitable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
     if let syn::Data::Union(_) = s.ast().data {
@@ -8,19 +8,21 @@ pub fn type_visitable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2:
 
     // ignore fields with #[type_visitable(ignore)]
     s.filter(|bi| {
-        !bi.ast()
-            .attrs
-            .iter()
-            .map(Attribute::parse_meta)
-            .filter_map(Result::ok)
-            .flat_map(|attr| match attr {
-                Meta::List(list) if list.path.is_ident("type_visitable") => list.nested,
-                _ => Default::default(),
-            })
-            .any(|nested| match nested {
-                NestedMeta::Meta(Meta::Path(path)) => path.is_ident("ignore"),
-                _ => false,
-            })
+        let mut ignored = false;
+
+        bi.ast().attrs.iter().for_each(|attr| {
+            if !attr.path().is_ident("type_visitable") {
+                return;
+            }
+            let _ = attr.parse_nested_meta(|nested| {
+                if nested.path.is_ident("ignore") {
+                    ignored = true;
+                }
+                Ok(())
+            });
+        });
+
+        !ignored
     });
 
     if !s.ast().generics.lifetimes().any(|lt| lt.lifetime.ident == "tcx") {
diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml
index bee5c8541..4d7c133e0 100644
--- a/compiler/rustc_metadata/Cargo.toml
+++ b/compiler/rustc_metadata/Cargo.toml
@@ -18,6 +18,7 @@ rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_feature = { path = "../rustc_feature" }
+rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_metadata/locales/en-US.ftl b/compiler/rustc_metadata/messages.ftl
index 79b8b4172..79b8b4172 100644
--- a/compiler/rustc_metadata/locales/en-US.ftl
+++ b/compiler/rustc_metadata/messages.ftl
diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs
index b05626311..23aceca06 100644
--- a/compiler/rustc_metadata/src/creader.rs
+++ b/compiler/rustc_metadata/src/creader.rs
@@ -6,11 +6,11 @@ use crate::rmeta::{CrateDep, CrateMetadata, CrateNumMap, CrateRoot, MetadataBlob
 
 use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_ast::{self as ast, *};
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::MappedReadGuard;
+use rustc_data_structures::sync::{MappedReadGuard, MappedWriteGuard, ReadGuard, WriteGuard};
 use rustc_expand::base::SyntaxExtension;
-use rustc_hir::def_id::{CrateNum, LocalDefId, StableCrateId, LOCAL_CRATE};
+use rustc_hir::def_id::{CrateNum, LocalDefId, StableCrateId, StableCrateIdMap, LOCAL_CRATE};
 use rustc_hir::definitions::Definitions;
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::TyCtxt;
@@ -46,9 +46,8 @@ pub struct CStore {
     /// This crate has a `#[alloc_error_handler]` item.
     has_alloc_error_handler: bool,
 
-    /// This map is used to verify we get no hash conflicts between
-    /// `StableCrateId` values.
-    pub(crate) stable_crate_ids: FxHashMap<StableCrateId, CrateNum>,
+    /// The interned [StableCrateId]s.
+    pub(crate) stable_crate_ids: StableCrateIdMap,
 
     /// Unused externs of the crate
     unused_externs: Vec<Symbol>,
@@ -133,14 +132,32 @@ impl<'a> std::fmt::Debug for CrateDump<'a> {
 
 impl CStore {
     pub fn from_tcx(tcx: TyCtxt<'_>) -> MappedReadGuard<'_, CStore> {
-        MappedReadGuard::map(tcx.cstore_untracked(), |c| {
-            c.as_any().downcast_ref::<CStore>().expect("`tcx.cstore` is not a `CStore`")
+        ReadGuard::map(tcx.untracked().cstore.read(), |cstore| {
+            cstore.as_any().downcast_ref::<CStore>().expect("`tcx.cstore` is not a `CStore`")
         })
     }
 
-    fn alloc_new_crate_num(&mut self) -> CrateNum {
-        self.metas.push(None);
-        CrateNum::new(self.metas.len() - 1)
+    pub fn from_tcx_mut(tcx: TyCtxt<'_>) -> MappedWriteGuard<'_, CStore> {
+        WriteGuard::map(tcx.untracked().cstore.write(), |cstore| {
+            cstore.untracked_as_any().downcast_mut().expect("`tcx.cstore` is not a `CStore`")
+        })
+    }
+
+    fn intern_stable_crate_id(&mut self, root: &CrateRoot) -> Result<CrateNum, CrateError> {
+        assert_eq!(self.metas.len(), self.stable_crate_ids.len());
+        let num = CrateNum::new(self.stable_crate_ids.len());
+        if let Some(&existing) = self.stable_crate_ids.get(&root.stable_crate_id()) {
+            let crate_name0 = root.name();
+            if let Some(crate_name1) = self.metas[existing].as_ref().map(|data| data.name()) {
+                Err(CrateError::StableCrateIdCollision(crate_name0, crate_name1))
+            } else {
+                Err(CrateError::SymbolConflictsCurrent(crate_name0))
+            }
+        } else {
+            self.metas.push(None);
+            self.stable_crate_ids.insert(root.stable_crate_id(), num);
+            Ok(num)
+        }
     }
 
     pub fn has_crate_data(&self, cnum: CrateNum) -> bool {
@@ -168,7 +185,7 @@ impl CStore {
     fn push_dependencies_in_postorder(&self, deps: &mut Vec<CrateNum>, cnum: CrateNum) {
         if !deps.contains(&cnum) {
             let data = self.get_crate_data(cnum);
-            for &dep in data.dependencies().iter() {
+            for dep in data.dependencies() {
                 if dep != cnum {
                     self.push_dependencies_in_postorder(deps, dep);
                 }
@@ -241,7 +258,7 @@ impl CStore {
     }
 
     pub fn new(sess: &Session) -> CStore {
-        let mut stable_crate_ids = FxHashMap::default();
+        let mut stable_crate_ids = StableCrateIdMap::default();
         stable_crate_ids.insert(sess.local_stable_crate_id(), LOCAL_CRATE);
         CStore {
             // We add an empty entry for LOCAL_CRATE (which maps to zero) in
@@ -268,9 +285,6 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
     ) -> Self {
         CrateLoader { tcx, cstore, used_extern_options }
     }
-    pub fn cstore(&self) -> &CStore {
-        &self.cstore
-    }
 
     fn existing_match(&self, name: Symbol, hash: Option<Svh>, kind: PathKind) -> Option<CrateNum> {
         for (cnum, data) in self.cstore.iter_crate_data() {
@@ -339,42 +353,6 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
         None
     }
 
-    fn verify_no_symbol_conflicts(&self, root: &CrateRoot) -> Result<(), CrateError> {
-        // Check for (potential) conflicts with the local crate
-        if self.sess.local_stable_crate_id() == root.stable_crate_id() {
-            return Err(CrateError::SymbolConflictsCurrent(root.name()));
-        }
-
-        // Check for conflicts with any crate loaded so far
-        for (_, other) in self.cstore.iter_crate_data() {
-            // Same stable crate id but different SVH
-            if other.stable_crate_id() == root.stable_crate_id() && other.hash() != root.hash() {
-                bug!(
-                    "Previously returned E0523 here. \
-                     See https://github.com/rust-lang/rust/pull/100599 for additional discussion.\
-                     root.name() = {}.",
-                    root.name()
-                );
-            }
-        }
-
-        Ok(())
-    }
-
-    fn verify_no_stable_crate_id_hash_conflicts(
-        &mut self,
-        root: &CrateRoot,
-        cnum: CrateNum,
-    ) -> Result<(), CrateError> {
-        if let Some(existing) = self.cstore.stable_crate_ids.insert(root.stable_crate_id(), cnum) {
-            let crate_name0 = root.name();
-            let crate_name1 = self.cstore.get_crate_data(existing).name();
-            return Err(CrateError::StableCrateIdCollision(crate_name0, crate_name1));
-        }
-
-        Ok(())
-    }
-
     fn register_crate(
         &mut self,
         host_lib: Option<Library>,
@@ -393,7 +371,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
             self.sess.opts.externs.get(name.as_str()).map_or(false, |e| e.is_private_dep);
 
         // Claim this crate number and cache it
-        let cnum = self.cstore.alloc_new_crate_num();
+        let cnum = self.cstore.intern_stable_crate_id(&crate_root)?;
 
         info!(
             "register crate `{}` (cnum = {}. private_dep = {})",
@@ -429,14 +407,6 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
             None
         };
 
-        // Perform some verification *after* resolve_crate_deps() above is
-        // known to have been successful. It seems that - in error cases - the
-        // cstore can be in a temporarily invalid state between cnum allocation
-        // and dependency resolution and the verification code would produce
-        // ICEs in that case (see #83045).
-        self.verify_no_symbol_conflicts(&crate_root)?;
-        self.verify_no_stable_crate_id_hash_conflicts(&crate_root, cnum)?;
-
         let crate_metadata = CrateMetadata::new(
             self.sess,
             &self.cstore,
@@ -635,7 +605,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
         if cmeta.update_extern_crate(extern_crate) {
             // Propagate the extern crate info to dependencies if it was updated.
             let extern_crate = ExternCrate { dependency_of: cnum, ..extern_crate };
-            for &dep_cnum in cmeta.dependencies().iter() {
+            for dep_cnum in cmeta.dependencies() {
                 self.update_extern_crate(dep_cnum, extern_crate);
             }
         }
@@ -717,8 +687,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
         // compilation mode also comes into play.
         let desired_strategy = self.sess.panic_strategy();
         let mut runtime_found = false;
-        let mut needs_panic_runtime =
-            self.sess.contains_name(&krate.attrs, sym::needs_panic_runtime);
+        let mut needs_panic_runtime = attr::contains_name(&krate.attrs, sym::needs_panic_runtime);
 
         for (cnum, data) in self.cstore.iter_crate_data() {
             needs_panic_runtime = needs_panic_runtime || data.needs_panic_runtime();
@@ -786,7 +755,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
         info!("loading profiler");
 
         let name = Symbol::intern(&self.sess.opts.unstable_opts.profiler_runtime);
-        if name == sym::profiler_builtins && self.sess.contains_name(&krate.attrs, sym::no_core) {
+        if name == sym::profiler_builtins && attr::contains_name(&krate.attrs, sym::no_core) {
             self.sess.emit_err(errors::ProfilerBuiltinsNeedsCore);
         }
 
@@ -800,14 +769,14 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
     }
 
     fn inject_allocator_crate(&mut self, krate: &ast::Crate) {
-        self.cstore.has_global_allocator = match &*global_allocator_spans(&self.sess, krate) {
+        self.cstore.has_global_allocator = match &*global_allocator_spans(krate) {
             [span1, span2, ..] => {
                 self.sess.emit_err(errors::NoMultipleGlobalAlloc { span2: *span2, span1: *span1 });
                 true
             }
             spans => !spans.is_empty(),
         };
-        self.cstore.has_alloc_error_handler = match &*alloc_error_handler_spans(&self.sess, krate) {
+        self.cstore.has_alloc_error_handler = match &*alloc_error_handler_spans(krate) {
             [span1, span2, ..] => {
                 self.sess
                     .emit_err(errors::NoMultipleAllocErrorHandler { span2: *span2, span1: *span1 });
@@ -819,7 +788,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
         // Check to see if we actually need an allocator. This desire comes
         // about through the `#![needs_allocator]` attribute and is typically
         // written down in liballoc.
-        if !self.sess.contains_name(&krate.attrs, sym::needs_allocator)
+        if !attr::contains_name(&krate.attrs, sym::needs_allocator)
             && !self.cstore.iter_crate_data().any(|(_, data)| data.needs_allocator())
         {
             return;
@@ -878,7 +847,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
             // allocator. At this point our allocator request is typically fulfilled
             // by the standard library, denoted by the `#![default_lib_allocator]`
             // attribute.
-            if !self.sess.contains_name(&krate.attrs, sym::default_lib_allocator)
+            if !attr::contains_name(&krate.attrs, sym::default_lib_allocator)
                 && !self.cstore.iter_crate_data().any(|(_, data)| data.has_default_lib_allocator())
             {
                 self.sess.emit_err(errors::GlobalAllocRequired);
@@ -1000,7 +969,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
                     }
                     None => item.ident.name,
                 };
-                let dep_kind = if self.sess.contains_name(&item.attrs, sym::no_link) {
+                let dep_kind = if attr::contains_name(&item.attrs, sym::no_link) {
                     CrateDepKind::MacrosOnly
                 } else {
                     CrateDepKind::Explicit
@@ -1046,16 +1015,15 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
     }
 }
 
-fn global_allocator_spans(sess: &Session, krate: &ast::Crate) -> Vec<Span> {
-    struct Finder<'a> {
-        sess: &'a Session,
+fn global_allocator_spans(krate: &ast::Crate) -> Vec<Span> {
+    struct Finder {
         name: Symbol,
         spans: Vec<Span>,
     }
-    impl<'ast, 'a> visit::Visitor<'ast> for Finder<'a> {
+    impl<'ast> visit::Visitor<'ast> for Finder {
         fn visit_item(&mut self, item: &'ast ast::Item) {
             if item.ident.name == self.name
-                && self.sess.contains_name(&item.attrs, sym::rustc_std_internal_symbol)
+                && attr::contains_name(&item.attrs, sym::rustc_std_internal_symbol)
             {
                 self.spans.push(item.span);
             }
@@ -1064,21 +1032,20 @@ fn global_allocator_spans(sess: &Session, krate: &ast::Crate) -> Vec<Span> {
     }
 
     let name = Symbol::intern(&AllocatorKind::Global.fn_name(sym::alloc));
-    let mut f = Finder { sess, name, spans: Vec::new() };
+    let mut f = Finder { name, spans: Vec::new() };
     visit::walk_crate(&mut f, krate);
     f.spans
 }
 
-fn alloc_error_handler_spans(sess: &Session, krate: &ast::Crate) -> Vec<Span> {
-    struct Finder<'a> {
-        sess: &'a Session,
+fn alloc_error_handler_spans(krate: &ast::Crate) -> Vec<Span> {
+    struct Finder {
         name: Symbol,
         spans: Vec<Span>,
     }
-    impl<'ast, 'a> visit::Visitor<'ast> for Finder<'a> {
+    impl<'ast> visit::Visitor<'ast> for Finder {
         fn visit_item(&mut self, item: &'ast ast::Item) {
             if item.ident.name == self.name
-                && self.sess.contains_name(&item.attrs, sym::rustc_std_internal_symbol)
+                && attr::contains_name(&item.attrs, sym::rustc_std_internal_symbol)
             {
                 self.spans.push(item.span);
             }
@@ -1087,7 +1054,7 @@ fn alloc_error_handler_spans(sess: &Session, krate: &ast::Crate) -> Vec<Span> {
     }
 
     let name = Symbol::intern(&AllocatorKind::Global.fn_name(sym::oom));
-    let mut f = Finder { sess, name, spans: Vec::new() };
+    let mut f = Finder { name, spans: Vec::new() };
     visit::walk_crate(&mut f, krate);
     f.spans
 }
diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs
index f64318997..08de828fb 100644
--- a/compiler/rustc_metadata/src/fs.rs
+++ b/compiler/rustc_metadata/src/fs.rs
@@ -6,9 +6,9 @@ use crate::{encode_metadata, EncodedMetadata};
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config::{CrateType, OutputType};
+use rustc_session::config::OutputType;
 use rustc_session::output::filename_for_metadata;
-use rustc_session::Session;
+use rustc_session::{MetadataKind, Session};
 use tempfile::Builder as TempFileBuilder;
 
 use std::fs;
@@ -39,27 +39,6 @@ pub fn emit_wrapper_file(
 }
 
 pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) {
-    #[derive(PartialEq, Eq, PartialOrd, Ord)]
-    enum MetadataKind {
-        None,
-        Uncompressed,
-        Compressed,
-    }
-
-    let metadata_kind = tcx
-        .sess
-        .crate_types()
-        .iter()
-        .map(|ty| match *ty {
-            CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None,
-
-            CrateType::Rlib => MetadataKind::Uncompressed,
-
-            CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed,
-        })
-        .max()
-        .unwrap_or(MetadataKind::None);
-
     let crate_name = tcx.crate_name(LOCAL_CRATE);
     let out_filename = filename_for_metadata(tcx.sess, crate_name, tcx.output_filenames(()));
     // To avoid races with another rustc process scanning the output directory,
@@ -76,6 +55,7 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) {
 
     // Always create a file at `metadata_filename`, even if we have nothing to write to it.
     // This simplifies the creation of the output `out_filename` when requested.
+    let metadata_kind = tcx.sess.metadata_kind();
     match metadata_kind {
         MetadataKind::None => {
             std::fs::File::create(&metadata_filename).unwrap_or_else(|err| {
diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs
index 6f6d3731c..81e62eccb 100644
--- a/compiler/rustc_metadata/src/lib.rs
+++ b/compiler/rustc_metadata/src/lib.rs
@@ -4,7 +4,6 @@
 #![feature(generators)]
 #![feature(iter_from_generator)]
 #![feature(let_chains)]
-#![feature(once_cell)]
 #![feature(proc_macro_internals)]
 #![feature(macro_metavar_expr)]
 #![feature(min_specialization)]
@@ -23,8 +22,6 @@ extern crate proc_macro;
 extern crate rustc_macros;
 #[macro_use]
 extern crate rustc_middle;
-#[macro_use]
-extern crate rustc_data_structures;
 
 #[macro_use]
 extern crate tracing;
@@ -47,4 +44,4 @@ pub use fs::{emit_wrapper_file, METADATA_FILENAME};
 pub use native_libs::find_native_static_library;
 pub use rmeta::{encode_metadata, EncodedMetadata, METADATA_HEADER};
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs
index 755a24253..c6af8d632 100644
--- a/compiler/rustc_metadata/src/locator.rs
+++ b/compiler/rustc_metadata/src/locator.rs
@@ -218,10 +218,11 @@ use crate::rmeta::{rustc_version, MetadataBlob, METADATA_HEADER};
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::memmap::Mmap;
-use rustc_data_structures::owning_ref::OwningRef;
+use rustc_data_structures::owned_slice::slice_owned;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_errors::{DiagnosticArgValue, FatalError, IntoDiagnosticArg};
+use rustc_fs_util::try_canonicalize;
 use rustc_session::config::{self, CrateType};
 use rustc_session::cstore::{CrateSource, MetadataLoader};
 use rustc_session::filesearch::FileSearch;
@@ -235,8 +236,9 @@ use rustc_target::spec::{Target, TargetTriple};
 use snap::read::FrameDecoder;
 use std::borrow::Cow;
 use std::io::{Read, Result as IoResult, Write};
+use std::ops::Deref;
 use std::path::{Path, PathBuf};
-use std::{cmp, fmt, fs};
+use std::{cmp, fmt};
 
 #[derive(Clone)]
 pub(crate) struct CrateLocator<'a> {
@@ -441,7 +443,7 @@ impl<'a> CrateLocator<'a> {
                 info!("lib candidate: {}", spf.path.display());
 
                 let (rlibs, rmetas, dylibs) = candidates.entry(hash.to_string()).or_default();
-                let path = fs::canonicalize(&spf.path).unwrap_or_else(|_| spf.path.clone());
+                let path = try_canonicalize(&spf.path).unwrap_or_else(|_| spf.path.clone());
                 if seen_paths.contains(&path) {
                     continue;
                 };
@@ -636,7 +638,7 @@ impl<'a> CrateLocator<'a> {
             // as well.
             if let Some((prev, _)) = &ret {
                 let sysroot = self.sysroot;
-                let sysroot = sysroot.canonicalize().unwrap_or_else(|_| sysroot.to_path_buf());
+                let sysroot = try_canonicalize(sysroot).unwrap_or_else(|_| sysroot.to_path_buf());
                 if prev.starts_with(&sysroot) {
                     continue;
                 }
@@ -760,14 +762,14 @@ impl<'a> CrateLocator<'a> {
     }
 
     pub(crate) fn into_error(self, root: Option<CratePaths>) -> CrateError {
-        CrateError::LocatorCombined(CombinedLocatorError {
+        CrateError::LocatorCombined(Box::new(CombinedLocatorError {
             crate_name: self.crate_name,
             root,
             triple: self.triple,
             dll_prefix: self.target.dll_prefix.to_string(),
             dll_suffix: self.target.dll_suffix.to_string(),
             crate_rejections: self.crate_rejections,
-        })
+        }))
     }
 }
 
@@ -789,6 +791,9 @@ fn get_metadata_section<'p>(
                 loader.get_dylib_metadata(target, filename).map_err(MetadataError::LoadFailure)?;
             // The header is uncompressed
             let header_len = METADATA_HEADER.len();
+            // header + u32 length of data
+            let data_start = header_len + 4;
+
             debug!("checking {} bytes of metadata-version stamp", header_len);
             let header = &buf[..cmp::min(header_len, buf.len())];
             if header != METADATA_HEADER {
@@ -798,21 +803,26 @@ fn get_metadata_section<'p>(
                 )));
             }
 
+            // Length of the compressed stream - this allows linkers to pad the section if they want
+            let Ok(len_bytes) = <[u8; 4]>::try_from(&buf[header_len..cmp::min(data_start, buf.len())]) else {
+                return Err(MetadataError::LoadFailure("invalid metadata length found".to_string()));
+            };
+            let compressed_len = u32::from_be_bytes(len_bytes) as usize;
+
             // Header is okay -> inflate the actual metadata
-            let compressed_bytes = &buf[header_len..];
+            let compressed_bytes = &buf[data_start..(data_start + compressed_len)];
             debug!("inflating {} bytes of compressed metadata", compressed_bytes.len());
             // Assume the decompressed data will be at least the size of the compressed data, so we
             // don't have to grow the buffer as much.
             let mut inflated = Vec::with_capacity(compressed_bytes.len());
-            match FrameDecoder::new(compressed_bytes).read_to_end(&mut inflated) {
-                Ok(_) => rustc_erase_owner!(OwningRef::new(inflated).map_owner_box()),
-                Err(_) => {
-                    return Err(MetadataError::LoadFailure(format!(
-                        "failed to decompress metadata: {}",
-                        filename.display()
-                    )));
-                }
-            }
+            FrameDecoder::new(compressed_bytes).read_to_end(&mut inflated).map_err(|_| {
+                MetadataError::LoadFailure(format!(
+                    "failed to decompress metadata: {}",
+                    filename.display()
+                ))
+            })?;
+
+            slice_owned(inflated, Deref::deref)
         }
         CrateFlavor::Rmeta => {
             // mmap the file, because only a small fraction of it is read.
@@ -830,7 +840,7 @@ fn get_metadata_section<'p>(
                 ))
             })?;
 
-            rustc_erase_owner!(OwningRef::new(mmap).map_owner_box())
+            slice_owned(mmap, Deref::deref)
         }
     };
     let blob = MetadataBlob::new(raw_bytes);
@@ -948,7 +958,7 @@ pub(crate) enum CrateError {
     StableCrateIdCollision(Symbol, Symbol),
     DlOpen(String),
     DlSym(String),
-    LocatorCombined(CombinedLocatorError),
+    LocatorCombined(Box<CombinedLocatorError>),
     NonDylibPlugin(Symbol),
 }
 
diff --git a/compiler/rustc_metadata/src/native_libs.rs b/compiler/rustc_metadata/src/native_libs.rs
index d6f68b2e1..b855c8e43 100644
--- a/compiler/rustc_metadata/src/native_libs.rs
+++ b/compiler/rustc_metadata/src/native_libs.rs
@@ -46,7 +46,7 @@ pub fn find_native_static_library(
 }
 
 fn find_bundled_library(
-    name: Option<Symbol>,
+    name: Symbol,
     verbatim: Option<bool>,
     kind: NativeLibKind,
     has_cfg: bool,
@@ -58,7 +58,7 @@ fn find_bundled_library(
     {
         let verbatim = verbatim.unwrap_or(false);
         let search_paths = &sess.target_filesearch(PathKind::Native).search_path_dirs();
-        return find_native_static_library(name.unwrap().as_str(), verbatim, search_paths, sess)
+        return find_native_static_library(name.as_str(), verbatim, search_paths, sess)
             .file_name()
             .and_then(|s| s.to_str())
             .map(Symbol::intern);
@@ -336,10 +336,16 @@ impl<'tcx> Collector<'tcx> {
                 if name.is_some() || kind.is_some() || modifiers.is_some() || cfg.is_some() {
                     sess.emit_err(errors::IncompatibleWasmLink { span });
                 }
-            } else if name.is_none() {
-                sess.emit_err(errors::LinkRequiresName { span: m.span });
             }
 
+            if wasm_import_module.is_some() {
+                (name, kind) = (wasm_import_module, Some(NativeLibKind::WasmImportModule));
+            }
+            let Some((name, name_span)) = name else {
+                sess.emit_err(errors::LinkRequiresName { span: m.span });
+                continue;
+            };
+
             // Do this outside of the loop so that `import_name_type` can be specified before `kind`.
             if let Some((_, span)) = import_name_type {
                 if kind != Some(NativeLibKind::RawDylib) {
@@ -349,8 +355,8 @@ impl<'tcx> Collector<'tcx> {
 
             let dll_imports = match kind {
                 Some(NativeLibKind::RawDylib) => {
-                    if let Some((name, span)) = name && name.as_str().contains('\0') {
-                        sess.emit_err(errors::RawDylibNoNul { span });
+                    if name.as_str().contains('\0') {
+                        sess.emit_err(errors::RawDylibNoNul { span: name_span });
                     }
                     foreign_mod_items
                         .iter()
@@ -389,7 +395,6 @@ impl<'tcx> Collector<'tcx> {
                 }
             };
 
-            let name = name.map(|(name, _)| name);
             let kind = kind.unwrap_or(NativeLibKind::Unspecified);
             let filename = find_bundled_library(name, verbatim, kind, cfg.is_some(), sess);
             self.libs.push(NativeLib {
@@ -398,7 +403,6 @@ impl<'tcx> Collector<'tcx> {
                 kind,
                 cfg,
                 foreign_module: Some(it.owner_id.to_def_id()),
-                wasm_import_module: wasm_import_module.map(|(name, _)| name),
                 verbatim,
                 dll_imports,
             });
@@ -415,11 +419,7 @@ impl<'tcx> Collector<'tcx> {
                 self.tcx.sess.emit_err(errors::LibFrameworkApple);
             }
             if let Some(ref new_name) = lib.new_name {
-                let any_duplicate = self
-                    .libs
-                    .iter()
-                    .filter_map(|lib| lib.name.as_ref())
-                    .any(|n| n.as_str() == lib.name);
+                let any_duplicate = self.libs.iter().any(|n| n.name.as_str() == lib.name);
                 if new_name.is_empty() {
                     self.tcx.sess.emit_err(errors::EmptyRenamingTarget { lib_name: &lib.name });
                 } else if !any_duplicate {
@@ -444,33 +444,28 @@ impl<'tcx> Collector<'tcx> {
             let mut existing = self
                 .libs
                 .drain_filter(|lib| {
-                    if let Some(lib_name) = lib.name {
-                        if lib_name.as_str() == passed_lib.name {
-                            // FIXME: This whole logic is questionable, whether modifiers are
-                            // involved or not, library reordering and kind overriding without
-                            // explicit `:rename` in particular.
-                            if lib.has_modifiers() || passed_lib.has_modifiers() {
-                                match lib.foreign_module {
-                                    Some(def_id) => {
-                                        self.tcx.sess.emit_err(errors::NoLinkModOverride {
-                                            span: Some(self.tcx.def_span(def_id)),
-                                        })
-                                    }
-                                    None => self
-                                        .tcx
-                                        .sess
-                                        .emit_err(errors::NoLinkModOverride { span: None }),
-                                };
-                            }
-                            if passed_lib.kind != NativeLibKind::Unspecified {
-                                lib.kind = passed_lib.kind;
-                            }
-                            if let Some(new_name) = &passed_lib.new_name {
-                                lib.name = Some(Symbol::intern(new_name));
-                            }
-                            lib.verbatim = passed_lib.verbatim;
-                            return true;
+                    if lib.name.as_str() == passed_lib.name {
+                        // FIXME: This whole logic is questionable, whether modifiers are
+                        // involved or not, library reordering and kind overriding without
+                        // explicit `:rename` in particular.
+                        if lib.has_modifiers() || passed_lib.has_modifiers() {
+                            match lib.foreign_module {
+                                Some(def_id) => self.tcx.sess.emit_err(errors::NoLinkModOverride {
+                                    span: Some(self.tcx.def_span(def_id)),
+                                }),
+                                None => {
+                                    self.tcx.sess.emit_err(errors::NoLinkModOverride { span: None })
+                                }
+                            };
+                        }
+                        if passed_lib.kind != NativeLibKind::Unspecified {
+                            lib.kind = passed_lib.kind;
+                        }
+                        if let Some(new_name) = &passed_lib.new_name {
+                            lib.name = Symbol::intern(new_name);
                         }
+                        lib.verbatim = passed_lib.verbatim;
+                        return true;
                     }
                     false
                 })
@@ -478,7 +473,7 @@ impl<'tcx> Collector<'tcx> {
             if existing.is_empty() {
                 // Add if not found
                 let new_name: Option<&str> = passed_lib.new_name.as_deref();
-                let name = Some(Symbol::intern(new_name.unwrap_or(&passed_lib.name)));
+                let name = Symbol::intern(new_name.unwrap_or(&passed_lib.name));
                 let sess = self.tcx.sess;
                 let filename =
                     find_bundled_library(name, passed_lib.verbatim, passed_lib.kind, false, sess);
@@ -488,7 +483,6 @@ impl<'tcx> Collector<'tcx> {
                     kind: passed_lib.kind,
                     cfg: None,
                     foreign_module: None,
-                    wasm_import_module: None,
                     verbatim: passed_lib.verbatim,
                     dll_imports: Vec::new(),
                 });
diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs
index b1e59b0a4..9f41dc92f 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder.rs
@@ -1,13 +1,14 @@
 // Decoding metadata from a single crate's metadata
 
 use crate::creader::{CStore, CrateMetadataRef};
+use crate::rmeta::table::IsDefault;
 use crate::rmeta::*;
 
 use rustc_ast as ast;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::{Lock, LockGuard, Lrc, OnceCell};
+use rustc_data_structures::sync::{AppendOnlyVec, Lock, Lrc, OnceCell};
 use rustc_data_structures::unhash::UnhashMap;
 use rustc_expand::base::{SyntaxExtension, SyntaxExtensionKind};
 use rustc_expand::proc_macro::{AttrProcMacro, BangProcMacro, DeriveProcMacro};
@@ -30,7 +31,6 @@ use rustc_session::cstore::{
 };
 use rustc_session::Session;
 use rustc_span::hygiene::ExpnIndex;
-use rustc_span::source_map::{respan, Spanned};
 use rustc_span::symbol::{kw, Ident, Symbol};
 use rustc_span::{self, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP};
 
@@ -52,12 +52,6 @@ mod cstore_impl;
 #[derive(Clone)]
 pub(crate) struct MetadataBlob(Lrc<MetadataRef>);
 
-// This is needed so we can create an OwningRef into the blob.
-// The data behind a `MetadataBlob` has a stable address because it is
-// contained within an Rc/Arc.
-unsafe impl rustc_data_structures::owning_ref::StableAddress for MetadataBlob {}
-
-// This is needed so we can create an OwningRef into the blob.
 impl std::ops::Deref for MetadataBlob {
     type Target = [u8];
 
@@ -110,7 +104,7 @@ pub(crate) struct CrateMetadata {
     /// IDs as they are seen from the current compilation session.
     cnum_map: CrateNumMap,
     /// Same ID set as `cnum_map` plus maybe some injected crates like panic runtime.
-    dependencies: Lock<Vec<CrateNum>>,
+    dependencies: AppendOnlyVec<CrateNum>,
     /// How to link (or not link) this crate to the currently compiled crate.
     dep_kind: Lock<CrateDepKind>,
     /// Filesystem location of this crate.
@@ -311,8 +305,11 @@ impl<T: ParameterizedOverTcx> LazyArray<T> {
 impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
     #[inline]
     fn tcx(&self) -> TyCtxt<'tcx> {
-        debug_assert!(self.tcx.is_some(), "missing TyCtxt in DecodeContext");
-        self.tcx.unwrap()
+        let Some(tcx) = self.tcx else {
+            bug!("No TyCtxt found for decoding. \
+                You need to explicitly pass `(crate_metadata_ref, tcx)` to `decode` instead of just `crate_metadata_ref`.");
+        };
+        tcx
     }
 
     #[inline]
@@ -454,7 +451,12 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for ast::AttrId {
 impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for SyntaxContext {
     fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> SyntaxContext {
         let cdata = decoder.cdata();
-        let sess = decoder.sess.unwrap();
+
+        let Some(sess) = decoder.sess else {
+            bug!("Cannot decode SyntaxContext without Session.\
+                You need to explicitly pass `(crate_metadata_ref, tcx)` to `decode` instead of just `crate_metadata_ref`.");
+        };
+
         let cname = cdata.root.name;
         rustc_span::hygiene::decode_syntax_context(decoder, &cdata.hygiene_context, |_, id| {
             debug!("SpecializedDecoder<SyntaxContext>: decoding {}", id);
@@ -471,7 +473,11 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for SyntaxContext {
 impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for ExpnId {
     fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> ExpnId {
         let local_cdata = decoder.cdata();
-        let sess = decoder.sess.unwrap();
+
+        let Some(sess) = decoder.sess else {
+            bug!("Cannot decode ExpnId without Session. \
+                You need to explicitly pass `(crate_metadata_ref, tcx)` to `decode` instead of just `crate_metadata_ref`.");
+        };
 
         let cnum = CrateNum::decode(decoder);
         let index = u32::decode(decoder);
@@ -520,7 +526,8 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for Span {
         let hi = lo + len;
 
         let Some(sess) = decoder.sess else {
-            bug!("Cannot decode Span without Session.")
+            bug!("Cannot decode Span without Session. \
+                You need to explicitly pass `(crate_metadata_ref, tcx)` to `decode` instead of just `crate_metadata_ref`.")
         };
 
         // Index of the file in the corresponding crate's list of encoded files.
@@ -743,6 +750,10 @@ impl CrateRoot {
 }
 
 impl<'a, 'tcx> CrateMetadataRef<'a> {
+    fn missing(self, descr: &str, id: DefIndex) -> ! {
+        bug!("missing `{descr}` for {:?}", self.local_def_id(id))
+    }
+
     fn raw_proc_macro(self, id: DefIndex) -> &'a ProcMacro {
         // DefIndex's in root.proc_macro_data have a one-to-one correspondence
         // with items in 'raw_proc_macros'.
@@ -776,8 +787,13 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
 
     fn opt_item_ident(self, item_index: DefIndex, sess: &Session) -> Option<Ident> {
         let name = self.opt_item_name(item_index)?;
-        let span =
-            self.root.tables.def_ident_span.get(self, item_index).unwrap().decode((self, sess));
+        let span = self
+            .root
+            .tables
+            .def_ident_span
+            .get(self, item_index)
+            .unwrap_or_else(|| self.missing("def_ident_span", item_index))
+            .decode((self, sess));
         Some(Ident::new(name, span))
     }
 
@@ -806,7 +822,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             .tables
             .def_span
             .get(self, index)
-            .unwrap_or_else(|| panic!("Missing span for {index:?}"))
+            .unwrap_or_else(|| self.missing("def_span", index))
             .decode((self, sess))
     }
 
@@ -841,7 +857,12 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         )
     }
 
-    fn get_variant(self, kind: &DefKind, index: DefIndex, parent_did: DefId) -> ty::VariantDef {
+    fn get_variant(
+        self,
+        kind: DefKind,
+        index: DefIndex,
+        parent_did: DefId,
+    ) -> (VariantIdx, ty::VariantDef) {
         let adt_kind = match kind {
             DefKind::Variant => ty::AdtKind::Enum,
             DefKind::Struct => ty::AdtKind::Struct,
@@ -855,27 +876,30 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             if adt_kind == ty::AdtKind::Enum { Some(self.local_def_id(index)) } else { None };
         let ctor = data.ctor.map(|(kind, index)| (kind, self.local_def_id(index)));
 
-        ty::VariantDef::new(
-            self.item_name(index),
-            variant_did,
-            ctor,
-            data.discr,
-            self.root
-                .tables
-                .children
-                .get(self, index)
-                .expect("fields are not encoded for a variant")
-                .decode(self)
-                .map(|index| ty::FieldDef {
-                    did: self.local_def_id(index),
-                    name: self.item_name(index),
-                    vis: self.get_visibility(index),
-                })
-                .collect(),
-            adt_kind,
-            parent_did,
-            false,
-            data.is_non_exhaustive,
+        (
+            data.idx,
+            ty::VariantDef::new(
+                self.item_name(index),
+                variant_did,
+                ctor,
+                data.discr,
+                self.root
+                    .tables
+                    .children
+                    .get(self, index)
+                    .expect("fields are not encoded for a variant")
+                    .decode(self)
+                    .map(|index| ty::FieldDef {
+                        did: self.local_def_id(index),
+                        name: self.item_name(index),
+                        vis: self.get_visibility(index),
+                    })
+                    .collect(),
+                adt_kind,
+                parent_did,
+                false,
+                data.is_non_exhaustive,
+            ),
         )
     }
 
@@ -891,7 +915,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         };
         let repr = self.root.tables.repr_options.get(self, item_id).unwrap().decode(self);
 
-        let variants = if let ty::AdtKind::Enum = adt_kind {
+        let mut variants: Vec<_> = if let ty::AdtKind::Enum = adt_kind {
             self.root
                 .tables
                 .children
@@ -902,27 +926,30 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                     let kind = self.def_kind(index);
                     match kind {
                         DefKind::Ctor(..) => None,
-                        _ => Some(self.get_variant(&kind, index, did)),
+                        _ => Some(self.get_variant(kind, index, did)),
                     }
                 })
                 .collect()
         } else {
-            std::iter::once(self.get_variant(&kind, item_id, did)).collect()
+            std::iter::once(self.get_variant(kind, item_id, did)).collect()
         };
 
-        tcx.mk_adt_def(did, adt_kind, variants, repr)
-    }
+        variants.sort_by_key(|(idx, _)| *idx);
 
-    fn get_generics(self, item_id: DefIndex, sess: &Session) -> ty::Generics {
-        self.root.tables.generics_of.get(self, item_id).unwrap().decode((self, sess))
+        tcx.mk_adt_def(
+            did,
+            adt_kind,
+            variants.into_iter().map(|(_, variant)| variant).collect(),
+            repr,
+        )
     }
 
-    fn get_visibility(self, id: DefIndex) -> ty::Visibility<DefId> {
+    fn get_visibility(self, id: DefIndex) -> Visibility<DefId> {
         self.root
             .tables
             .visibility
             .get(self, id)
-            .unwrap()
+            .unwrap_or_else(|| self.missing("visibility", id))
             .decode(self)
             .map_id(|index| self.local_def_id(index))
     }
@@ -932,7 +959,12 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     }
 
     fn get_expn_that_defined(self, id: DefIndex, sess: &Session) -> ExpnId {
-        self.root.tables.expn_that_defined.get(self, id).unwrap().decode((self, sess))
+        self.root
+            .tables
+            .expn_that_defined
+            .get(self, id)
+            .unwrap_or_else(|| self.missing("expn_that_defined", id))
+            .decode((self, sess))
     }
 
     fn get_debugger_visualizers(self) -> Vec<rustc_span::DebuggerVisualizerFile> {
@@ -979,17 +1011,11 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
 
     fn get_mod_child(self, id: DefIndex, sess: &Session) -> ModChild {
         let ident = self.item_ident(id, sess);
-        let kind = self.def_kind(id);
-        let def_id = self.local_def_id(id);
-        let res = Res::Def(kind, def_id);
+        let res = Res::Def(self.def_kind(id), self.local_def_id(id));
         let vis = self.get_visibility(id);
         let span = self.get_span(id, sess);
-        let macro_rules = match kind {
-            DefKind::Macro(..) => self.root.tables.is_macro_rules.get(self, id),
-            _ => false,
-        };
 
-        ModChild { ident, res, vis, span, macro_rules }
+        ModChild { ident, res, vis, span, reexport_chain: Default::default() }
     }
 
     /// Iterates over all named children of the given module,
@@ -1013,10 +1039,14 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             } else {
                 // Iterate over all children.
                 for child_index in self.root.tables.children.get(self, id).unwrap().decode(self) {
-                    yield self.get_mod_child(child_index, sess);
+                    // FIXME: Do not encode RPITITs as a part of this list.
+                    if self.root.tables.opt_rpitit_info.get(self, child_index).is_none() {
+                        yield self.get_mod_child(child_index, sess);
+                    }
                 }
 
-                if let Some(reexports) = self.root.tables.module_reexports.get(self, id) {
+                let reexports = self.root.tables.module_children_reexports.get(self, id);
+                if !reexports.is_default() {
                     for reexport in reexports.decode((self, sess)) {
                         yield reexport;
                     }
@@ -1033,13 +1063,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         self.root.tables.optimized_mir.get(self, id).is_some()
     }
 
-    fn module_expansion(self, id: DefIndex, sess: &Session) -> ExpnId {
-        match self.def_kind(id) {
-            DefKind::Mod | DefKind::Enum | DefKind::Trait => self.get_expn_that_defined(id, sess),
-            _ => panic!("Expected module, found {:?}", self.local_def_id(id)),
-        }
-    }
-
     fn get_fn_has_self_parameter(self, id: DefIndex, sess: &'a Session) -> bool {
         self.root
             .tables
@@ -1066,8 +1089,11 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     }
 
     fn get_associated_item(self, id: DefIndex, sess: &'a Session) -> ty::AssocItem {
-        let name = self.item_name(id);
-
+        let name = if self.root.tables.opt_rpitit_info.get(self, id).is_some() {
+            kw::Empty
+        } else {
+            self.item_name(id)
+        };
         let (kind, has_self) = match self.def_kind(id) {
             DefKind::AssocConst => (ty::AssocKind::Const, false),
             DefKind::AssocFn => (ty::AssocKind::Fn, self.get_fn_has_self_parameter(id, sess)),
@@ -1075,6 +1101,8 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             _ => bug!("cannot get associated-item of `{:?}`", self.def_key(id)),
         };
         let container = self.root.tables.assoc_container.get(self, id).unwrap();
+        let opt_rpitit_info =
+            self.root.tables.opt_rpitit_info.get(self, id).map(|d| d.decode(self));
 
         ty::AssocItem {
             name,
@@ -1083,6 +1111,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             trait_item_def_id: self.get_trait_item_def_id(id),
             container,
             fn_has_self_parameter: has_self,
+            opt_rpitit_info,
         }
     }
 
@@ -1121,33 +1150,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             .decode((self, sess))
     }
 
-    fn get_struct_field_names(
-        self,
-        id: DefIndex,
-        sess: &'a Session,
-    ) -> impl Iterator<Item = Spanned<Symbol>> + 'a {
-        self.root
-            .tables
-            .children
-            .get(self, id)
-            .expect("fields not encoded for a struct")
-            .decode(self)
-            .map(move |index| respan(self.get_span(index, sess), self.item_name(index)))
-    }
-
-    fn get_struct_field_visibilities(
-        self,
-        id: DefIndex,
-    ) -> impl Iterator<Item = Visibility<DefId>> + 'a {
-        self.root
-            .tables
-            .children
-            .get(self, id)
-            .expect("fields not encoded for a struct")
-            .decode(self)
-            .map(move |field_index| self.get_visibility(field_index))
-    }
-
     fn get_inherent_implementations_for_type(
         self,
         tcx: TyCtxt<'tcx>,
@@ -1612,7 +1614,7 @@ impl CrateMetadata {
             .collect();
         let alloc_decoding_state =
             AllocDecodingState::new(root.interpret_alloc_index.decode(&blob).collect());
-        let dependencies = Lock::new(cnum_map.iter().cloned().collect());
+        let dependencies = cnum_map.iter().copied().collect();
 
         // Pre-decode the DefPathHash->DefIndex table. This is a cheap operation
         // that does not copy any data. It just does some data verification.
@@ -1652,12 +1654,12 @@ impl CrateMetadata {
         cdata
     }
 
-    pub(crate) fn dependencies(&self) -> LockGuard<'_, Vec<CrateNum>> {
-        self.dependencies.borrow()
+    pub(crate) fn dependencies(&self) -> impl Iterator<Item = CrateNum> + '_ {
+        self.dependencies.iter()
     }
 
     pub(crate) fn add_dependency(&self, cnum: CrateNum) {
-        self.dependencies.borrow_mut().push(cnum);
+        self.dependencies.push(cnum);
     }
 
     pub(crate) fn update_extern_crate(&self, new_extern_crate: ExternCrate) -> bool {
@@ -1721,10 +1723,6 @@ impl CrateMetadata {
         self.root.name
     }
 
-    pub(crate) fn stable_crate_id(&self) -> StableCrateId {
-        self.root.stable_crate_id
-    }
-
     pub(crate) fn hash(&self) -> Svh {
         self.root.hash
     }
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
index 83a0e833e..31798afb8 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -13,14 +13,15 @@ use rustc_middle::arena::ArenaAllocatable;
 use rustc_middle::metadata::ModChild;
 use rustc_middle::middle::exported_symbols::ExportedSymbol;
 use rustc_middle::middle::stability::DeprecationEntry;
+use rustc_middle::query::LocalCrate;
 use rustc_middle::ty::fast_reject::SimplifiedType;
 use rustc_middle::ty::query::{ExternProviders, Providers};
-use rustc_middle::ty::{self, TyCtxt, Visibility};
-use rustc_session::cstore::{CrateSource, CrateStore};
+use rustc_middle::ty::{self, TyCtxt};
+use rustc_session::cstore::CrateStore;
 use rustc_session::{Session, StableCrateId};
 use rustc_span::hygiene::{ExpnHash, ExpnId};
-use rustc_span::source_map::{Span, Spanned};
 use rustc_span::symbol::{kw, Symbol};
+use rustc_span::Span;
 
 use rustc_data_structures::sync::Lrc;
 use std::any::Any;
@@ -226,7 +227,7 @@ provide! { tcx, def_id, other, cdata,
     lookup_default_body_stability => { table }
     lookup_deprecation_entry => { table }
     params_in_repr => { table }
-    unused_generic_params => { table }
+    unused_generic_params => { cdata.root.tables.unused_generic_params.get(cdata, def_id.index) }
     opt_def_kind => { table_direct }
     impl_parent => { table }
     impl_polarity => { table_direct }
@@ -252,9 +253,21 @@ provide! { tcx, def_id, other, cdata,
             .get(cdata, def_id.index)
             .map(|lazy| lazy.decode((cdata, tcx)))
             .process_decoded(tcx, || panic!("{def_id:?} does not have trait_impl_trait_tys")))
-     }
+    }
+    implied_predicates_of => {
+        cdata
+            .root
+            .tables
+            .implied_predicates_of
+            .get(cdata, def_id.index)
+            .map(|lazy| lazy.decode((cdata, tcx)))
+            .unwrap_or_else(|| {
+                debug_assert_eq!(tcx.def_kind(def_id), DefKind::Trait);
+                tcx.super_predicates_of(def_id)
+            })
+    }
 
-    associated_items_for_impl_trait_in_trait => { table_defaulted_array }
+    associated_types_for_impl_traits_in_associated_fn => { table_defaulted_array }
 
     visibility => { cdata.get_visibility(def_id.index) }
     adt_def => { cdata.get_adt_def(def_id.index, tcx) }
@@ -367,10 +380,7 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) {
     *providers = Providers {
         allocator_kind: |tcx, ()| CStore::from_tcx(tcx).allocator_kind(),
         alloc_error_handler_kind: |tcx, ()| CStore::from_tcx(tcx).alloc_error_handler_kind(),
-        is_private_dep: |_tcx, cnum| {
-            assert_eq!(cnum, LOCAL_CRATE);
-            false
-        },
+        is_private_dep: |_tcx, LocalCrate| false,
         native_library: |tcx, id| {
             tcx.native_libraries(id.krate)
                 .iter()
@@ -386,12 +396,8 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) {
                         .contains(&id)
                 })
         },
-        native_libraries: |tcx, cnum| {
-            assert_eq!(cnum, LOCAL_CRATE);
-            native_libs::collect(tcx)
-        },
-        foreign_modules: |tcx, cnum| {
-            assert_eq!(cnum, LOCAL_CRATE);
+        native_libraries: |tcx, LocalCrate| native_libs::collect(tcx),
+        foreign_modules: |tcx, LocalCrate| {
             foreign_modules::collect(tcx).into_iter().map(|m| (m.def_id, m)).collect()
         },
 
@@ -489,47 +495,27 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) {
         },
 
         dependency_formats: |tcx, ()| Lrc::new(crate::dependency_format::calculate(tcx)),
-        has_global_allocator: |tcx, cnum| {
-            assert_eq!(cnum, LOCAL_CRATE);
-            CStore::from_tcx(tcx).has_global_allocator()
-        },
-        has_alloc_error_handler: |tcx, cnum| {
-            assert_eq!(cnum, LOCAL_CRATE);
-            CStore::from_tcx(tcx).has_alloc_error_handler()
-        },
+        has_global_allocator: |tcx, LocalCrate| CStore::from_tcx(tcx).has_global_allocator(),
+        has_alloc_error_handler: |tcx, LocalCrate| CStore::from_tcx(tcx).has_alloc_error_handler(),
         postorder_cnums: |tcx, ()| {
             tcx.arena
                 .alloc_slice(&CStore::from_tcx(tcx).crate_dependencies_in_postorder(LOCAL_CRATE))
         },
-        crates: |tcx, ()| tcx.arena.alloc_from_iter(CStore::from_tcx(tcx).crates_untracked()),
+        crates: |tcx, ()| {
+            // The list of loaded crates is now frozen in query cache,
+            // so make sure cstore is not mutably accessed from here on.
+            tcx.untracked().cstore.leak();
+            tcx.arena.alloc_from_iter(CStore::from_tcx(tcx).iter_crate_data().map(|(cnum, _)| cnum))
+        },
         ..*providers
     };
 }
 
 impl CStore {
-    pub fn struct_field_names_untracked<'a>(
-        &'a self,
-        def: DefId,
-        sess: &'a Session,
-    ) -> impl Iterator<Item = Spanned<Symbol>> + 'a {
-        self.get_crate_data(def.krate).get_struct_field_names(def.index, sess)
-    }
-
-    pub fn struct_field_visibilities_untracked(
-        &self,
-        def: DefId,
-    ) -> impl Iterator<Item = Visibility<DefId>> + '_ {
-        self.get_crate_data(def.krate).get_struct_field_visibilities(def.index)
-    }
-
     pub fn ctor_untracked(&self, def: DefId) -> Option<(CtorKind, DefId)> {
         self.get_crate_data(def.krate).get_ctor(def.index)
     }
 
-    pub fn visibility_untracked(&self, def: DefId) -> Visibility<DefId> {
-        self.get_crate_data(def.krate).get_visibility(def.index)
-    }
-
     pub fn module_children_untracked<'a>(
         &'a self,
         def_id: DefId,
@@ -566,32 +552,16 @@ impl CStore {
         )
     }
 
-    pub fn fn_has_self_parameter_untracked(&self, def: DefId, sess: &Session) -> bool {
-        self.get_crate_data(def.krate).get_fn_has_self_parameter(def.index, sess)
-    }
-
-    pub fn crate_source_untracked(&self, cnum: CrateNum) -> Lrc<CrateSource> {
-        self.get_crate_data(cnum).source.clone()
-    }
-
-    pub fn get_span_untracked(&self, def_id: DefId, sess: &Session) -> Span {
+    pub fn def_span_untracked(&self, def_id: DefId, sess: &Session) -> Span {
         self.get_crate_data(def_id.krate).get_span(def_id.index, sess)
     }
 
-    pub fn def_kind(&self, def: DefId) -> DefKind {
+    pub fn def_kind_untracked(&self, def: DefId) -> DefKind {
         self.get_crate_data(def.krate).def_kind(def.index)
     }
 
-    pub fn crates_untracked(&self) -> impl Iterator<Item = CrateNum> + '_ {
-        self.iter_crate_data().map(|(cnum, _)| cnum)
-    }
-
-    pub fn item_generics_num_lifetimes(&self, def_id: DefId, sess: &Session) -> usize {
-        self.get_crate_data(def_id.krate).get_generics(def_id.index, sess).own_counts().lifetimes
-    }
-
-    pub fn module_expansion_untracked(&self, def_id: DefId, sess: &Session) -> ExpnId {
-        self.get_crate_data(def_id.krate).module_expansion(def_id.index, sess)
+    pub fn expn_that_defined_untracked(&self, def_id: DefId, sess: &Session) -> ExpnId {
+        self.get_crate_data(def_id.krate).get_expn_that_defined(def_id.index, sess)
     }
 
     /// Only public-facing way to traverse all the definitions in a non-local crate.
@@ -601,14 +571,6 @@ impl CStore {
         self.get_crate_data(cnum).num_def_ids()
     }
 
-    pub fn item_attrs_untracked<'a>(
-        &'a self,
-        def_id: DefId,
-        sess: &'a Session,
-    ) -> impl Iterator<Item = ast::Attribute> + 'a {
-        self.get_crate_data(def_id.krate).get_item_attrs(def_id.index, sess)
-    }
-
     pub fn get_proc_macro_quoted_span_untracked(
         &self,
         cnum: CrateNum,
@@ -636,7 +598,10 @@ impl CrateStore for CStore {
     }
 
     fn stable_crate_id_to_crate_num(&self, stable_crate_id: StableCrateId) -> CrateNum {
-        self.stable_crate_ids[&stable_crate_id]
+        *self
+            .stable_crate_ids
+            .get(&stable_crate_id)
+            .unwrap_or_else(|| bug!("uninterned StableCrateId: {stable_crate_id:?}"))
     }
 
     /// Returns the `DefKey` for a given `DefId`. This indicates the
diff --git a/compiler/rustc_metadata/src/rmeta/def_path_hash_map.rs b/compiler/rustc_metadata/src/rmeta/def_path_hash_map.rs
index a6133f1b4..02cab561b 100644
--- a/compiler/rustc_metadata/src/rmeta/def_path_hash_map.rs
+++ b/compiler/rustc_metadata/src/rmeta/def_path_hash_map.rs
@@ -1,14 +1,14 @@
 use crate::rmeta::DecodeContext;
 use crate::rmeta::EncodeContext;
-use crate::rmeta::MetadataBlob;
-use rustc_data_structures::owning_ref::OwningRef;
+use rustc_data_structures::owned_slice::slice_owned;
+use rustc_data_structures::owned_slice::OwnedSlice;
 use rustc_hir::def_path_hash_map::{Config as HashMapConfig, DefPathHashMap};
 use rustc_middle::parameterized_over_tcx;
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use rustc_span::def_id::{DefIndex, DefPathHash};
 
 pub(crate) enum DefPathHashMapRef<'tcx> {
-    OwnedFromMetadata(odht::HashTable<HashMapConfig, OwningRef<MetadataBlob, [u8]>>),
+    OwnedFromMetadata(odht::HashTable<HashMapConfig, OwnedSlice>),
     BorrowedFromTcx(&'tcx DefPathHashMap),
 }
 
@@ -50,11 +50,11 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for DefPathHashMapRef<'static>
 
         let len = d.read_usize();
         let pos = d.position();
-        let o = OwningRef::new(d.blob().clone()).map(|x| &x[pos..pos + len]);
+        let o = slice_owned(d.blob().clone(), |blob| &blob[pos..pos + len]);
 
-        // Although we already have the data we need via the OwningRef, we still need
-        // to advance the DecodeContext's position so it's in a valid state after
-        // the method. We use read_raw_bytes() for that.
+        // Although we already have the data we need via the `OwnedSlice`, we still need
+        // to advance the `DecodeContext`'s position so it's in a valid state after
+        // the method. We use `read_raw_bytes()` for that.
         let _ = d.read_raw_bytes(len);
 
         let inner = odht::HashTable::from_raw_bytes(o).unwrap_or_else(|e| {
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index 3ab01f780..e44b133a9 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -24,6 +24,7 @@ use rustc_middle::middle::exported_symbols::{
     metadata_symbol_name, ExportedSymbol, SymbolExportInfo,
 };
 use rustc_middle::mir::interpret;
+use rustc_middle::query::LocalCrate;
 use rustc_middle::traits::specialization_graph;
 use rustc_middle::ty::codec::TyEncoder;
 use rustc_middle::ty::fast_reject::{self, SimplifiedType, TreatParams};
@@ -42,7 +43,6 @@ use std::borrow::Borrow;
 use std::collections::hash_map::Entry;
 use std::hash::Hash;
 use std::io::{Read, Seek, Write};
-use std::iter;
 use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 
@@ -111,8 +111,6 @@ impl<'a, 'tcx> Encoder for EncodeContext<'a, 'tcx> {
         emit_i8(i8);
 
         emit_bool(bool);
-        emit_f64(f64);
-        emit_f32(f32);
         emit_char(char);
         emit_str(&str);
         emit_raw_bytes(&[u8]);
@@ -457,7 +455,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     }
 
     fn encode_info_for_items(&mut self) {
-        self.encode_info_for_mod(CRATE_DEF_ID, self.tcx.hir().root_module());
+        self.encode_info_for_mod(CRATE_DEF_ID);
 
         // Proc-macro crates only export proc-macro items, which are looked
         // up using `proc_macro_data`
@@ -609,10 +607,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
         _ = stat!("mir", || self.encode_mir());
 
-        _ = stat!("items", || {
-            self.encode_def_ids();
-            self.encode_info_for_items();
-        });
+        _ = stat!("def-ids", || self.encode_def_ids());
+
+        _ = stat!("items", || self.encode_info_for_items());
 
         let interpret_alloc_index = stat!("interpret-alloc-index", || {
             let mut interpret_alloc_index = Vec::new();
@@ -681,17 +678,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 has_global_allocator: tcx.has_global_allocator(LOCAL_CRATE),
                 has_alloc_error_handler: tcx.has_alloc_error_handler(LOCAL_CRATE),
                 has_panic_handler: tcx.has_panic_handler(LOCAL_CRATE),
-                has_default_lib_allocator: tcx
-                    .sess
-                    .contains_name(&attrs, sym::default_lib_allocator),
+                has_default_lib_allocator: attr::contains_name(&attrs, sym::default_lib_allocator),
                 proc_macro_data,
                 debugger_visualizers,
-                compiler_builtins: tcx.sess.contains_name(&attrs, sym::compiler_builtins),
-                needs_allocator: tcx.sess.contains_name(&attrs, sym::needs_allocator),
-                needs_panic_runtime: tcx.sess.contains_name(&attrs, sym::needs_panic_runtime),
-                no_builtins: tcx.sess.contains_name(&attrs, sym::no_builtins),
-                panic_runtime: tcx.sess.contains_name(&attrs, sym::panic_runtime),
-                profiler_runtime: tcx.sess.contains_name(&attrs, sym::profiler_runtime),
+                compiler_builtins: attr::contains_name(&attrs, sym::compiler_builtins),
+                needs_allocator: attr::contains_name(&attrs, sym::needs_allocator),
+                needs_panic_runtime: attr::contains_name(&attrs, sym::needs_panic_runtime),
+                no_builtins: attr::contains_name(&attrs, sym::no_builtins),
+                panic_runtime: attr::contains_name(&attrs, sym::panic_runtime),
+                profiler_runtime: attr::contains_name(&attrs, sym::profiler_runtime),
                 symbol_mangling_version: tcx.sess.opts.get_symbol_mangling_version(),
 
                 crate_deps,
@@ -815,7 +810,7 @@ fn analyze_attr(attr: &Attribute, state: &mut AnalyzeAttrState) -> bool {
     should_encode
 }
 
-fn should_encode_visibility(def_kind: DefKind) -> bool {
+fn should_encode_span(def_kind: DefKind) -> bool {
     match def_kind {
         DefKind::Mod
         | DefKind::Struct
@@ -827,25 +822,136 @@ fn should_encode_visibility(def_kind: DefKind) -> bool {
         | DefKind::ForeignTy
         | DefKind::TraitAlias
         | DefKind::AssocTy
+        | DefKind::TyParam
+        | DefKind::ConstParam
+        | DefKind::LifetimeParam
         | DefKind::Fn
         | DefKind::Const
-        | DefKind::Static(..)
+        | DefKind::Static(_)
         | DefKind::Ctor(..)
         | DefKind::AssocFn
         | DefKind::AssocConst
-        | DefKind::Macro(..)
+        | DefKind::Macro(_)
+        | DefKind::AnonConst
+        | DefKind::InlineConst
+        | DefKind::OpaqueTy
+        | DefKind::Field
+        | DefKind::Impl { .. }
+        | DefKind::Closure
+        | DefKind::Generator => true,
+        DefKind::ExternCrate
         | DefKind::Use
         | DefKind::ForeignMod
+        | DefKind::ImplTraitPlaceholder
+        | DefKind::GlobalAsm => false,
+    }
+}
+
+fn should_encode_attrs(def_kind: DefKind) -> bool {
+    match def_kind {
+        DefKind::Mod
+        | DefKind::Struct
+        | DefKind::Union
+        | DefKind::Enum
+        | DefKind::Variant
+        | DefKind::Trait
+        | DefKind::TyAlias
+        | DefKind::ForeignTy
+        | DefKind::TraitAlias
+        | DefKind::AssocTy
+        | DefKind::Fn
+        | DefKind::Const
+        | DefKind::Static(_)
+        | DefKind::AssocFn
+        | DefKind::AssocConst
+        | DefKind::Macro(_)
+        | DefKind::Field
+        | DefKind::Impl { .. } => true,
+        DefKind::TyParam
+        | DefKind::ConstParam
+        | DefKind::Ctor(..)
+        | DefKind::ExternCrate
+        | DefKind::Use
+        | DefKind::ForeignMod
+        | DefKind::AnonConst
+        | DefKind::InlineConst
         | DefKind::OpaqueTy
         | DefKind::ImplTraitPlaceholder
-        | DefKind::Impl { .. }
+        | DefKind::LifetimeParam
+        | DefKind::GlobalAsm
+        | DefKind::Closure
+        | DefKind::Generator => false,
+    }
+}
+
+fn should_encode_expn_that_defined(def_kind: DefKind) -> bool {
+    match def_kind {
+        DefKind::Mod
+        | DefKind::Struct
+        | DefKind::Union
+        | DefKind::Enum
+        | DefKind::Variant
+        | DefKind::Trait
+        | DefKind::Impl { .. } => true,
+        DefKind::TyAlias
+        | DefKind::ForeignTy
+        | DefKind::TraitAlias
+        | DefKind::AssocTy
+        | DefKind::TyParam
+        | DefKind::Fn
+        | DefKind::Const
+        | DefKind::ConstParam
+        | DefKind::Static(_)
+        | DefKind::Ctor(..)
+        | DefKind::AssocFn
+        | DefKind::AssocConst
+        | DefKind::Macro(_)
+        | DefKind::ExternCrate
+        | DefKind::Use
+        | DefKind::ForeignMod
+        | DefKind::AnonConst
+        | DefKind::InlineConst
+        | DefKind::OpaqueTy
+        | DefKind::ImplTraitPlaceholder
+        | DefKind::Field
+        | DefKind::LifetimeParam
+        | DefKind::GlobalAsm
+        | DefKind::Closure
+        | DefKind::Generator => false,
+    }
+}
+
+fn should_encode_visibility(def_kind: DefKind) -> bool {
+    match def_kind {
+        DefKind::Mod
+        | DefKind::Struct
+        | DefKind::Union
+        | DefKind::Enum
+        | DefKind::Variant
+        | DefKind::Trait
+        | DefKind::TyAlias
+        | DefKind::ForeignTy
+        | DefKind::TraitAlias
+        | DefKind::AssocTy
+        | DefKind::Fn
+        | DefKind::Const
+        | DefKind::Static(..)
+        | DefKind::Ctor(..)
+        | DefKind::AssocFn
+        | DefKind::AssocConst
+        | DefKind::Macro(..)
         | DefKind::Field => true,
-        DefKind::TyParam
+        DefKind::Use
+        | DefKind::ForeignMod
+        | DefKind::TyParam
         | DefKind::ConstParam
         | DefKind::LifetimeParam
         | DefKind::AnonConst
         | DefKind::InlineConst
+        | DefKind::OpaqueTy
+        | DefKind::ImplTraitPlaceholder
         | DefKind::GlobalAsm
+        | DefKind::Impl { .. }
         | DefKind::Closure
         | DefKind::Generator
         | DefKind::ExternCrate => false,
@@ -1016,7 +1122,6 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
         | DefKind::Const
         | DefKind::Static(..)
         | DefKind::TyAlias
-        | DefKind::OpaqueTy
         | DefKind::ForeignTy
         | DefKind::Impl { .. }
         | DefKind::AssocFn
@@ -1027,8 +1132,20 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
         | DefKind::AnonConst
         | DefKind::InlineConst => true,
 
+        DefKind::OpaqueTy => {
+            let opaque = tcx.hir().expect_item(def_id).expect_opaque_ty();
+            if let hir::OpaqueTyOrigin::FnReturn(fn_def_id) | hir::OpaqueTyOrigin::AsyncFn(fn_def_id) = opaque.origin
+                && let hir::Node::TraitItem(trait_item) = tcx.hir().get_by_def_id(fn_def_id)
+                && let (_, hir::TraitFn::Required(..)) = trait_item.expect_fn()
+            {
+                false
+            } else {
+                true
+            }
+        }
+
         DefKind::ImplTraitPlaceholder => {
-            let parent_def_id = tcx.impl_trait_in_trait_parent(def_id.to_def_id());
+            let parent_def_id = tcx.impl_trait_in_trait_parent_fn(def_id.to_def_id());
             let assoc_item = tcx.associated_item(parent_def_id);
             match assoc_item.container {
                 // Always encode an RPIT in an impl fn, since it always has a body
@@ -1044,7 +1161,13 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
             let assoc_item = tcx.associated_item(def_id);
             match assoc_item.container {
                 ty::AssocItemContainer::ImplContainer => true,
-                ty::AssocItemContainer::TraitContainer => assoc_item.defaultness(tcx).has_value(),
+                // FIXME(-Zlower-impl-trait-in-trait-to-assoc-ty) always encode RPITITs,
+                // since we need to be able to "project" from an RPITIT associated item
+                // to an opaque when installing the default projection predicates in
+                // default trait methods with RPITITs.
+                ty::AssocItemContainer::TraitContainer => {
+                    assoc_item.defaultness(tcx).has_value() || assoc_item.opt_rpitit_info.is_some()
+                }
             }
         }
         DefKind::TyParam => {
@@ -1104,7 +1227,7 @@ fn should_encode_const(def_kind: DefKind) -> bool {
 // We only encode impl trait in trait when using `lower-impl-trait-in-trait-to-assoc-ty` unstable
 // option.
 fn should_encode_fn_impl_trait_in_trait<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
-    if tcx.sess.opts.unstable_opts.lower_impl_trait_in_trait_to_assoc_ty
+    if tcx.lower_impl_trait_in_trait_to_assoc_ty()
         && let Some(assoc_item) = tcx.opt_associated_item(def_id)
         && assoc_item.container == ty::AssocItemContainer::TraitContainer
         && assoc_item.kind == ty::AssocKind::Fn
@@ -1147,11 +1270,17 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             let def_kind = tcx.opt_def_kind(local_id);
             let Some(def_kind) = def_kind else { continue };
             self.tables.opt_def_kind.set_some(def_id.index, def_kind);
-            let def_span = tcx.def_span(local_id);
-            record!(self.tables.def_span[def_id] <- def_span);
-            self.encode_attrs(local_id);
-            record!(self.tables.expn_that_defined[def_id] <- self.tcx.expn_that_defined(def_id));
-            if let Some(ident_span) = tcx.def_ident_span(def_id) {
+            if should_encode_span(def_kind) {
+                let def_span = tcx.def_span(local_id);
+                record!(self.tables.def_span[def_id] <- def_span);
+            }
+            if should_encode_attrs(def_kind) {
+                self.encode_attrs(local_id);
+            }
+            if should_encode_expn_that_defined(def_kind) {
+                record!(self.tables.expn_that_defined[def_id] <- self.tcx.expn_that_defined(def_id));
+            }
+            if should_encode_span(def_kind) && let Some(ident_span) = tcx.def_ident_span(def_id) {
                 record!(self.tables.def_ident_span[def_id] <- ident_span);
             }
             if def_kind.has_codegen_attrs() {
@@ -1186,11 +1315,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 let default = self.tcx.object_lifetime_default(def_id);
                 record!(self.tables.object_lifetime_default[def_id] <- default);
             }
-            if let DefKind::Trait | DefKind::TraitAlias = def_kind {
+            if let DefKind::Trait = def_kind {
                 record!(self.tables.super_predicates_of[def_id] <- self.tcx.super_predicates_of(def_id));
             }
+            if let DefKind::TraitAlias = def_kind {
+                record!(self.tables.super_predicates_of[def_id] <- self.tcx.super_predicates_of(def_id));
+                record!(self.tables.implied_predicates_of[def_id] <- self.tcx.implied_predicates_of(def_id));
+            }
             if let DefKind::Enum | DefKind::Struct | DefKind::Union = def_kind {
-                self.encode_info_for_adt(def_id);
+                self.encode_info_for_adt(local_id);
             }
             if tcx.impl_method_has_trait_impl_trait_tys(def_id)
                 && let Ok(table) = self.tcx.collect_return_position_impl_trait_in_trait_tys(def_id)
@@ -1198,8 +1331,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record!(self.tables.trait_impl_trait_tys[def_id] <- table);
             }
             if should_encode_fn_impl_trait_in_trait(tcx, def_id) {
-                let table = tcx.associated_items_for_impl_trait_in_trait(def_id);
-                record_defaulted_array!(self.tables.associated_items_for_impl_trait_in_trait[def_id] <- table);
+                let table = tcx.associated_types_for_impl_traits_in_associated_fn(def_id);
+                record_defaulted_array!(self.tables.associated_types_for_impl_traits_in_associated_fn[def_id] <- table);
             }
         }
 
@@ -1223,7 +1356,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     }
 
     #[instrument(level = "trace", skip(self))]
-    fn encode_info_for_adt(&mut self, def_id: DefId) {
+    fn encode_info_for_adt(&mut self, local_def_id: LocalDefId) {
+        let def_id = local_def_id.to_def_id();
         let tcx = self.tcx;
         let adt_def = tcx.adt_def(def_id);
         record!(self.tables.repr_options[def_id] <- adt_def.repr());
@@ -1232,15 +1366,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         record!(self.tables.params_in_repr[def_id] <- params_in_repr);
 
         if adt_def.is_enum() {
-            record_array!(self.tables.children[def_id] <- iter::from_generator(||
-                for variant in tcx.adt_def(def_id).variants() {
-                    yield variant.def_id.index;
-                    // Encode constructors which take a separate slot in value namespace.
-                    if let Some(ctor_def_id) = variant.ctor_def_id() {
-                        yield ctor_def_id.index;
-                    }
-                }
-            ));
+            let module_children = tcx.module_children_non_reexports(local_def_id);
+            record_array!(self.tables.children[def_id] <-
+                module_children.iter().map(|def_id| def_id.local_def_index));
         } else {
             // For non-enum, there is only one variant, and its def_id is the adt's.
             debug_assert_eq!(adt_def.variants().len(), 1);
@@ -1248,9 +1376,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             // Therefore, the loop over variants will encode its fields as the adt's children.
         }
 
-        for variant in adt_def.variants().iter() {
+        for (idx, variant) in adt_def.variants().iter_enumerated() {
             let data = VariantData {
                 discr: variant.discr,
+                idx,
                 ctor: variant.ctor.map(|(kind, def_id)| (kind, def_id.index)),
                 is_non_exhaustive: variant.is_field_list_non_exhaustive(),
             };
@@ -1272,7 +1401,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         }
     }
 
-    fn encode_info_for_mod(&mut self, local_def_id: LocalDefId, md: &hir::Mod<'_>) {
+    fn encode_info_for_mod(&mut self, local_def_id: LocalDefId) {
         let tcx = self.tcx;
         let def_id = local_def_id.to_def_id();
         debug!("EncodeContext::encode_info_for_mod({:?})", def_id);
@@ -1286,38 +1415,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             // Encode this here because we don't do it in encode_def_ids.
             record!(self.tables.expn_that_defined[def_id] <- tcx.expn_that_defined(local_def_id));
         } else {
-            record_array!(self.tables.children[def_id] <- iter::from_generator(|| {
-                for item_id in md.item_ids {
-                    match tcx.hir().item(*item_id).kind {
-                        // Foreign items are planted into their parent modules
-                        // from name resolution point of view.
-                        hir::ItemKind::ForeignMod { items, .. } => {
-                            for foreign_item in items {
-                                yield foreign_item.id.owner_id.def_id.local_def_index;
-                            }
-                        }
-                        // Only encode named non-reexport children, reexports are encoded
-                        // separately and unnamed items are not used by name resolution.
-                        hir::ItemKind::ExternCrate(..) => continue,
-                        hir::ItemKind::Struct(ref vdata, _) => {
-                            yield item_id.owner_id.def_id.local_def_index;
-                            // Encode constructors which take a separate slot in value namespace.
-                            if let Some(ctor_def_id) = vdata.ctor_def_id() {
-                                yield ctor_def_id.local_def_index;
-                            }
-                        }
-                        _ if tcx.def_key(item_id.owner_id.to_def_id()).get_opt_name().is_some() => {
-                            yield item_id.owner_id.def_id.local_def_index;
-                        }
-                        _ => continue,
-                    }
-                }
-            }));
+            let non_reexports = tcx.module_children_non_reexports(local_def_id);
+            record_array!(self.tables.children[def_id] <-
+                non_reexports.iter().map(|def_id| def_id.local_def_index));
 
-            if let Some(reexports) = tcx.module_reexports(local_def_id) {
-                assert!(!reexports.is_empty());
-                record_array!(self.tables.module_reexports[def_id] <- reexports);
-            }
+            record_defaulted_array!(self.tables.module_children_reexports[def_id] <-
+                tcx.module_children_reexports(local_def_id));
         }
     }
 
@@ -1350,19 +1453,24 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         if trait_item.kind == ty::AssocKind::Fn {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
         }
+        if let Some(rpitit_info) = trait_item.opt_rpitit_info {
+            let rpitit_info = self.lazy(rpitit_info);
+            self.tables.opt_rpitit_info.set_some(def_id.index, rpitit_info);
+        }
     }
 
     fn encode_info_for_impl_item(&mut self, def_id: DefId) {
         debug!("EncodeContext::encode_info_for_impl_item({:?})", def_id);
         let tcx = self.tcx;
 
-        let ast_item = self.tcx.hir().expect_impl_item(def_id.expect_local());
-        self.tables.impl_defaultness.set_some(def_id.index, ast_item.defaultness);
+        let defaultness = self.tcx.impl_defaultness(def_id.expect_local());
+        self.tables.impl_defaultness.set_some(def_id.index, defaultness);
         let impl_item = self.tcx.associated_item(def_id);
         self.tables.assoc_container.set_some(def_id.index, impl_item.container);
 
         match impl_item.kind {
             ty::AssocKind::Fn => {
+                let ast_item = self.tcx.hir().expect_impl_item(def_id.expect_local());
                 let hir::ImplItemKind::Fn(ref sig, body) = ast_item.kind else { bug!() };
                 self.tables.asyncness.set_some(def_id.index, sig.header.asyncness);
                 record_array!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
@@ -1383,6 +1491,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
             self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id));
         }
+        if let Some(rpitit_info) = impl_item.opt_rpitit_info {
+            let rpitit_info = self.lazy(rpitit_info);
+            self.tables.opt_rpitit_info.set_some(def_id.index, rpitit_info);
+        }
     }
 
     fn encode_mir(&mut self) {
@@ -1431,9 +1543,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             let instance =
                 ty::InstanceDef::Item(ty::WithOptConstParam::unknown(def_id.to_def_id()));
             let unused = tcx.unused_generic_params(instance);
-            if !unused.all_used() {
-                record!(self.tables.unused_generic_params[def_id.to_def_id()] <- unused);
-            }
+            self.tables.unused_generic_params.set(def_id.local_def_index, unused);
         }
 
         // Encode all the deduced parameter attributes for everything that has MIR, even for items
@@ -1503,23 +1613,32 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         })
     }
 
-    fn encode_info_for_item(&mut self, def_id: DefId, item: &'tcx hir::Item<'tcx>) {
+    fn encode_info_for_item(&mut self, item: &'tcx hir::Item<'tcx>) {
         let tcx = self.tcx;
-
+        let def_id = item.owner_id.to_def_id();
         debug!("EncodeContext::encode_info_for_item({:?})", def_id);
 
+        let record_associated_item_def_ids = |this: &mut Self, def_ids: &[DefId]| {
+            record_array!(this.tables.children[def_id] <- def_ids.iter().map(|&def_id| {
+                assert!(def_id.is_local());
+                def_id.index
+            }))
+        };
+
         match item.kind {
             hir::ItemKind::Fn(ref sig, .., body) => {
                 self.tables.asyncness.set_some(def_id.index, sig.header.asyncness);
                 record_array!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
                 self.tables.constness.set_some(def_id.index, sig.header.constness);
+                record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
+                self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id));
             }
             hir::ItemKind::Macro(ref macro_def, _) => {
                 self.tables.is_macro_rules.set(def_id.index, macro_def.macro_rules);
                 record!(self.tables.macro_definition[def_id] <- &*macro_def.body);
             }
-            hir::ItemKind::Mod(ref m) => {
-                return self.encode_info_for_mod(item.owner_id.def_id, m);
+            hir::ItemKind::Mod(..) => {
+                self.encode_info_for_mod(item.owner_id.def_id);
             }
             hir::ItemKind::OpaqueTy(ref opaque) => {
                 self.encode_explicit_item_bounds(def_id);
@@ -1530,9 +1649,11 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             hir::ItemKind::Impl(hir::Impl { defaultness, constness, .. }) => {
                 self.tables.impl_defaultness.set_some(def_id.index, *defaultness);
                 self.tables.constness.set_some(def_id.index, *constness);
+                self.tables.impl_polarity.set_some(def_id.index, self.tcx.impl_polarity(def_id));
+
+                if let Some(trait_ref) = self.tcx.impl_trait_ref(def_id) {
+                    record!(self.tables.impl_trait_ref[def_id] <- trait_ref);
 
-                let trait_ref = self.tcx.impl_trait_ref(def_id);
-                if let Some(trait_ref) = trait_ref {
                     let trait_ref = trait_ref.skip_binder();
                     let trait_def = self.tcx.trait_def(trait_ref.def_id);
                     if let Ok(mut an) = trait_def.ancestors(self.tcx, def_id) {
@@ -1550,21 +1671,27 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                     }
                 }
 
-                let polarity = self.tcx.impl_polarity(def_id);
-                self.tables.impl_polarity.set_some(def_id.index, polarity);
+                let associated_item_def_ids = self.tcx.associated_item_def_ids(def_id);
+                record_associated_item_def_ids(self, associated_item_def_ids);
+                for &trait_item_def_id in associated_item_def_ids {
+                    self.encode_info_for_impl_item(trait_item_def_id);
+                }
             }
             hir::ItemKind::Trait(..) => {
-                let trait_def = self.tcx.trait_def(def_id);
-                record!(self.tables.trait_def[def_id] <- trait_def);
+                record!(self.tables.trait_def[def_id] <- self.tcx.trait_def(def_id));
+
+                let associated_item_def_ids = self.tcx.associated_item_def_ids(def_id);
+                record_associated_item_def_ids(self, associated_item_def_ids);
+                for &item_def_id in associated_item_def_ids {
+                    self.encode_info_for_trait_item(item_def_id);
+                }
             }
             hir::ItemKind::TraitAlias(..) => {
-                let trait_def = self.tcx.trait_def(def_id);
-                record!(self.tables.trait_def[def_id] <- trait_def);
+                record!(self.tables.trait_def[def_id] <- self.tcx.trait_def(def_id));
             }
-            hir::ItemKind::ExternCrate(_) | hir::ItemKind::Use(..) => {
-                bug!("cannot encode info for item {:?}", item)
-            }
-            hir::ItemKind::Static(..)
+            hir::ItemKind::ExternCrate(_)
+            | hir::ItemKind::Use(..)
+            | hir::ItemKind::Static(..)
             | hir::ItemKind::Const(..)
             | hir::ItemKind::Enum(..)
             | hir::ItemKind::Struct(..)
@@ -1572,49 +1699,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             | hir::ItemKind::ForeignMod { .. }
             | hir::ItemKind::GlobalAsm(..)
             | hir::ItemKind::TyAlias(..) => {}
-        };
-        // FIXME(eddyb) there should be a nicer way to do this.
-        match item.kind {
-            hir::ItemKind::Impl { .. } | hir::ItemKind::Trait(..) => {
-                let associated_item_def_ids = self.tcx.associated_item_def_ids(def_id);
-                record_array!(self.tables.children[def_id] <-
-                    associated_item_def_ids.iter().map(|&def_id| {
-                        assert!(def_id.is_local());
-                        def_id.index
-                    })
-                );
-            }
-            _ => {}
-        }
-        if let hir::ItemKind::Fn(..) = item.kind {
-            record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
-            self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id));
-        }
-        if let hir::ItemKind::Impl { .. } = item.kind {
-            if let Some(trait_ref) = self.tcx.impl_trait_ref(def_id) {
-                record!(self.tables.impl_trait_ref[def_id] <- trait_ref);
-            }
-        }
-        // In some cases, along with the item itself, we also
-        // encode some sub-items. Usually we want some info from the item
-        // so it's easier to do that here then to wait until we would encounter
-        // normally in the visitor walk.
-        match item.kind {
-            hir::ItemKind::Impl { .. } => {
-                for &trait_item_def_id in
-                    self.tcx.associated_item_def_ids(item.owner_id.to_def_id()).iter()
-                {
-                    self.encode_info_for_impl_item(trait_item_def_id);
-                }
-            }
-            hir::ItemKind::Trait(..) => {
-                for &item_def_id in
-                    self.tcx.associated_item_def_ids(item.owner_id.to_def_id()).iter()
-                {
-                    self.encode_info_for_trait_item(item_def_id);
-                }
-            }
-            _ => {}
         }
     }
 
@@ -1690,8 +1774,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             let stability = tcx.lookup_stability(CRATE_DEF_ID);
             let macros =
                 self.lazy_array(tcx.resolutions(()).proc_macros.iter().map(|p| p.local_def_index));
-            let spans = self.tcx.sess.parse_sess.proc_macro_quoted_spans();
-            for (i, span) in spans.into_iter().enumerate() {
+            for (i, span) in self.tcx.sess.parse_sess.proc_macro_quoted_spans() {
                 let span = self.lazy(span);
                 self.tables.proc_macro_quoted_spans.set_some(i, span);
             }
@@ -1723,11 +1806,11 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 // Proc-macros may have attributes like `#[allow_internal_unstable]`,
                 // so downstream crates need access to them.
                 let attrs = hir.attrs(proc_macro);
-                let macro_kind = if tcx.sess.contains_name(attrs, sym::proc_macro) {
+                let macro_kind = if attr::contains_name(attrs, sym::proc_macro) {
                     MacroKind::Bang
-                } else if tcx.sess.contains_name(attrs, sym::proc_macro_attribute) {
+                } else if attr::contains_name(attrs, sym::proc_macro_attribute) {
                     MacroKind::Attr
-                } else if let Some(attr) = tcx.sess.find_by_name(attrs, sym::proc_macro_derive) {
+                } else if let Some(attr) = attr::find_by_name(attrs, sym::proc_macro_derive) {
                     // This unwrap chain should have been checked by the proc-macro harness.
                     name = attr.meta_item_list().unwrap()[0]
                         .meta_item()
@@ -1858,7 +1941,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                     let simplified_self_ty = fast_reject::simplify_type(
                         self.tcx,
                         trait_ref.self_ty(),
-                        TreatParams::AsInfer,
+                        TreatParams::AsCandidateKey,
                     );
 
                     fx_hash_map
@@ -2001,10 +2084,7 @@ impl<'a, 'tcx> Visitor<'tcx> for EncodeContext<'a, 'tcx> {
     }
     fn visit_item(&mut self, item: &'tcx hir::Item<'tcx>) {
         intravisit::walk_item(self, item);
-        match item.kind {
-            hir::ItemKind::ExternCrate(_) | hir::ItemKind::Use(..) => {} // ignore these
-            _ => self.encode_info_for_item(item.owner_id.to_def_id(), item),
-        }
+        self.encode_info_for_item(item);
     }
     fn visit_foreign_item(&mut self, ni: &'tcx hir::ForeignItem<'tcx>) {
         intravisit::walk_foreign_item(self, ni);
@@ -2050,13 +2130,13 @@ fn prefetch_mir(tcx: TyCtxt<'_>) {
         let (encode_const, encode_opt) = should_encode_mir(tcx, def_id);
 
         if encode_const {
-            tcx.ensure().mir_for_ctfe(def_id);
+            tcx.ensure_with_value().mir_for_ctfe(def_id);
         }
         if encode_opt {
-            tcx.ensure().optimized_mir(def_id);
+            tcx.ensure_with_value().optimized_mir(def_id);
         }
         if encode_opt || encode_const {
-            tcx.ensure().promoted_mir(def_id);
+            tcx.ensure_with_value().promoted_mir(def_id);
         }
     })
 }
@@ -2224,18 +2304,16 @@ pub fn provide(providers: &mut Providers) {
         doc_link_resolutions: |tcx, def_id| {
             tcx.resolutions(())
                 .doc_link_resolutions
-                .get(&def_id.expect_local())
+                .get(&def_id)
                 .expect("no resolutions for a doc link")
         },
         doc_link_traits_in_scope: |tcx, def_id| {
             tcx.resolutions(())
                 .doc_link_traits_in_scope
-                .get(&def_id.expect_local())
+                .get(&def_id)
                 .expect("no traits in scope for a doc link")
         },
-        traits_in_crate: |tcx, cnum| {
-            assert_eq!(cnum, LOCAL_CRATE);
-
+        traits_in_crate: |tcx, LocalCrate| {
             let mut traits = Vec::new();
             for id in tcx.hir().items() {
                 if matches!(tcx.def_kind(id.owner_id), DefKind::Trait | DefKind::TraitAlias) {
@@ -2247,9 +2325,7 @@ pub fn provide(providers: &mut Providers) {
             traits.sort_by_cached_key(|&def_id| tcx.def_path_hash(def_id));
             tcx.arena.alloc_slice(&traits)
         },
-        trait_impls_in_crate: |tcx, cnum| {
-            assert_eq!(cnum, LOCAL_CRATE);
-
+        trait_impls_in_crate: |tcx, LocalCrate| {
             let mut trait_impls = Vec::new();
             for id in tcx.hir().items() {
                 if matches!(tcx.def_kind(id.owner_id), DefKind::Impl { .. })
diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs
index a7ec2d790..67710054c 100644
--- a/compiler/rustc_metadata/src/rmeta/mod.rs
+++ b/compiler/rustc_metadata/src/rmeta/mod.rs
@@ -31,6 +31,7 @@ use rustc_span::edition::Edition;
 use rustc_span::hygiene::{ExpnIndex, MacroKind};
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{self, ExpnData, ExpnHash, ExpnId, Span};
+use rustc_target::abi::VariantIdx;
 use rustc_target::spec::{PanicStrategy, TargetTriple};
 
 use std::marker::PhantomData;
@@ -55,13 +56,13 @@ pub(crate) fn rustc_version() -> String {
 /// Metadata encoding version.
 /// N.B., increment this if you change the format of metadata such that
 /// the rustc version can't be found to compare with `rustc_version()`.
-const METADATA_VERSION: u8 = 6;
+const METADATA_VERSION: u8 = 7;
 
 /// Metadata header which includes `METADATA_VERSION`.
 ///
-/// This header is followed by the position of the `CrateRoot`,
-/// which is encoded as a 32-bit big-endian unsigned integer,
-/// and further followed by the rustc version string.
+/// This header is followed by the length of the compressed data, then
+/// the position of the `CrateRoot`, which is encoded as a 32-bit big-endian
+/// unsigned integer, and further followed by the rustc version string.
 pub const METADATA_HEADER: &[u8] = &[b'r', b'u', b's', b't', 0, 0, 0, METADATA_VERSION];
 
 /// A value of type T referred to by its absolute position
@@ -354,7 +355,10 @@ define_tables! {
     explicit_item_bounds: Table<DefIndex, LazyArray<(ty::Predicate<'static>, Span)>>,
     inferred_outlives_of: Table<DefIndex, LazyArray<(ty::Clause<'static>, Span)>>,
     inherent_impls: Table<DefIndex, LazyArray<DefIndex>>,
-    associated_items_for_impl_trait_in_trait: Table<DefIndex, LazyArray<DefId>>,
+    associated_types_for_impl_traits_in_associated_fn: Table<DefIndex, LazyArray<DefId>>,
+    opt_rpitit_info: Table<DefIndex, Option<LazyValue<ty::ImplTraitInTraitData>>>,
+    unused_generic_params: Table<DefIndex, UnusedGenericParams>,
+    module_children_reexports: Table<DefIndex, LazyArray<ModChild>>,
 
 - optional:
     attributes: Table<DefIndex, LazyArray<ast::Attribute>>,
@@ -370,6 +374,9 @@ define_tables! {
     explicit_predicates_of: Table<DefIndex, LazyValue<ty::GenericPredicates<'static>>>,
     generics_of: Table<DefIndex, LazyValue<ty::Generics>>,
     super_predicates_of: Table<DefIndex, LazyValue<ty::GenericPredicates<'static>>>,
+    // As an optimization, we only store this for trait aliases,
+    // since it's identical to super_predicates_of for traits.
+    implied_predicates_of: Table<DefIndex, LazyValue<ty::GenericPredicates<'static>>>,
     type_of: Table<DefIndex, LazyValue<ty::EarlyBinder<Ty<'static>>>>,
     variances_of: Table<DefIndex, LazyArray<ty::Variance>>,
     fn_sig: Table<DefIndex, LazyValue<ty::EarlyBinder<ty::PolyFnSig<'static>>>>,
@@ -381,7 +388,6 @@ define_tables! {
     mir_for_ctfe: Table<DefIndex, LazyValue<mir::Body<'static>>>,
     mir_generator_witnesses: Table<DefIndex, LazyValue<mir::GeneratorLayout<'static>>>,
     promoted_mir: Table<DefIndex, LazyValue<IndexVec<mir::Promoted, mir::Body<'static>>>>,
-    // FIXME(compiler-errors): Why isn't this a LazyArray?
     thir_abstract_const: Table<DefIndex, LazyValue<ty::Const<'static>>>,
     impl_parent: Table<DefIndex, RawDefId>,
     impl_polarity: Table<DefIndex, ty::ImplPolarity>,
@@ -397,7 +403,6 @@ define_tables! {
     trait_def: Table<DefIndex, LazyValue<ty::TraitDef>>,
     trait_item_def_id: Table<DefIndex, RawDefId>,
     expn_that_defined: Table<DefIndex, LazyValue<ExpnId>>,
-    unused_generic_params: Table<DefIndex, LazyValue<UnusedGenericParams>>,
     params_in_repr: Table<DefIndex, LazyValue<BitSet<u32>>>,
     repr_options: Table<DefIndex, LazyValue<ReprOptions>>,
     // `def_keys` and `def_path_hashes` represent a lazy version of a
@@ -411,7 +416,6 @@ define_tables! {
     assoc_container: Table<DefIndex, ty::AssocItemContainer>,
     macro_definition: Table<DefIndex, LazyValue<ast::DelimArgs>>,
     proc_macro: Table<DefIndex, MacroKind>,
-    module_reexports: Table<DefIndex, LazyArray<ModChild>>,
     deduced_param_attrs: Table<DefIndex, LazyArray<DeducedParamAttrs>>,
     trait_impl_trait_tys: Table<DefIndex, LazyValue<FxHashMap<DefId, Ty<'static>>>>,
     doc_link_resolutions: Table<DefIndex, LazyValue<DocLinkResMap>>,
@@ -420,6 +424,7 @@ define_tables! {
 
 #[derive(TyEncodable, TyDecodable)]
 struct VariantData {
+    idx: VariantIdx,
     discr: ty::VariantDiscr,
     /// If this is unit or tuple-variant/struct, then this is the index of the ctor id.
     ctor: Option<(CtorKind, DefIndex)>,
diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs
index b89d48ec1..364fa74ab 100644
--- a/compiler/rustc_metadata/src/rmeta/table.rs
+++ b/compiler/rustc_metadata/src/rmeta/table.rs
@@ -3,7 +3,7 @@ use crate::rmeta::*;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_hir::def::{CtorKind, CtorOf};
 use rustc_index::vec::Idx;
-use rustc_middle::ty::ParameterizedOverTcx;
+use rustc_middle::ty::{ParameterizedOverTcx, UnusedGenericParams};
 use rustc_serialize::opaque::FileEncoder;
 use rustc_serialize::Encoder as _;
 use rustc_span::hygiene::MacroKind;
@@ -50,6 +50,16 @@ impl IsDefault for DefPathHash {
     }
 }
 
+impl IsDefault for UnusedGenericParams {
+    fn is_default(&self) -> bool {
+        // UnusedGenericParams encodes the *un*usedness as a bitset.
+        // This means that 0 corresponds to all bits used, which is indeed the default.
+        let is_default = self.bits() == 0;
+        debug_assert_eq!(is_default, self.all_used());
+        is_default
+    }
+}
+
 /// Helper trait, for encoding to, and decoding from, a fixed number of bytes.
 /// Used mainly for Lazy positions and lengths.
 /// Unchecked invariant: `Self::default()` should encode as `[0; BYTE_LEN]`,
@@ -271,6 +281,21 @@ impl FixedSizeEncoding for bool {
     }
 }
 
+impl FixedSizeEncoding for UnusedGenericParams {
+    type ByteArray = [u8; 4];
+
+    #[inline]
+    fn from_bytes(b: &[u8; 4]) -> Self {
+        let x: u32 = u32::from_bytes(b);
+        UnusedGenericParams::from_bits(x)
+    }
+
+    #[inline]
+    fn write_to_bytes(self, b: &mut [u8; 4]) {
+        self.bits().write_to_bytes(b);
+    }
+}
+
 // NOTE(eddyb) there could be an impl for `usize`, which would enable a more
 // generic `LazyValue<T>` impl, but in the general case we might not need / want
 // to fit every `usize` in `u32`.
diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml
index a2b78cc29..5b2ec9029 100644
--- a/compiler/rustc_middle/Cargo.toml
+++ b/compiler/rustc_middle/Cargo.toml
@@ -26,8 +26,8 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_query_system = { path = "../rustc_query_system" }
-rustc-rayon-core = { version = "0.4.0", optional = true }
-rustc-rayon = { version = "0.4.0", optional = true }
+rustc-rayon-core = { version = "0.5.0", optional = true }
+rustc-rayon = { version = "0.5.0", optional = true }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
diff --git a/compiler/rustc_middle/locales/en-US.ftl b/compiler/rustc_middle/messages.ftl
index 4f4e5c6a2..bd9d89dee 100644
--- a/compiler/rustc_middle/locales/en-US.ftl
+++ b/compiler/rustc_middle/messages.ftl
@@ -16,6 +16,10 @@ middle_limit_invalid =
     `limit` must be a non-negative integer
     .label = {$error_str}
 
+middle_recursion_limit_reached =
+    reached the recursion limit finding the struct tail for `{$ty}`
+    .help = consider increasing the recursion limit by adding a `#![recursion_limit = "{$suggested_limit}"]`
+
 middle_const_eval_non_int =
     constant evaluation of enum discriminant resulted in non-integer
 
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs
index 62e44b629..dd1e254f4 100644
--- a/compiler/rustc_middle/src/arena.rs
+++ b/compiler/rustc_middle/src/arena.rs
@@ -36,7 +36,7 @@ macro_rules! arena_types {
             )>,
             [] output_filenames: std::sync::Arc<rustc_session::config::OutputFilenames>,
             [] metadata_loader: rustc_data_structures::steal::Steal<Box<rustc_session::cstore::MetadataLoaderDyn>>,
-            [] crate_for_resolver: rustc_data_structures::steal::Steal<rustc_ast::ast::Crate>,
+            [] crate_for_resolver: rustc_data_structures::steal::Steal<(rustc_ast::Crate, rustc_ast::AttrVec)>,
             [] resolutions: rustc_middle::ty::ResolverGlobalCtxt,
             [decode] unsafety_check_result: rustc_middle::mir::UnsafetyCheckResult,
             [decode] code_region: rustc_middle::mir::coverage::CodeRegion,
@@ -94,7 +94,8 @@ macro_rules! arena_types {
             [] object_safety_violations: rustc_middle::traits::ObjectSafetyViolation,
             [] codegen_unit: rustc_middle::mir::mono::CodegenUnit<'tcx>,
             [decode] attribute: rustc_ast::Attribute,
-            [] name_set: rustc_data_structures::fx::FxHashSet<rustc_span::symbol::Symbol>,
+            [] name_set: rustc_data_structures::unord::UnordSet<rustc_span::symbol::Symbol>,
+            [] ordered_name_set: rustc_data_structures::fx::FxIndexSet<rustc_span::symbol::Symbol>,
             [] hir_id_set: rustc_hir::HirIdSet,
 
             // Interned types
@@ -107,6 +108,7 @@ macro_rules! arena_types {
             // (during lowering) and the `librustc_middle` arena (for decoding MIR)
             [decode] asm_template: rustc_ast::InlineAsmTemplatePiece,
             [decode] used_trait_imports: rustc_data_structures::unord::UnordSet<rustc_hir::def_id::LocalDefId>,
+            [decode] registered_tools: rustc_middle::ty::RegisteredTools,
             [decode] is_late_bound_map: rustc_data_structures::fx::FxIndexSet<rustc_hir::ItemLocalId>,
             [decode] impl_source: rustc_middle::traits::ImplSource<'tcx, ()>,
 
@@ -117,6 +119,7 @@ macro_rules! arena_types {
             [] external_constraints: rustc_middle::traits::solve::ExternalConstraintsData<'tcx>,
             [decode] doc_link_resolutions: rustc_hir::def::DocLinkResMap,
             [] closure_kind_origin: (rustc_span::Span, rustc_middle::hir::place::Place<'tcx>),
+            [] mod_child: rustc_middle::metadata::ModChild,
         ]);
     )
 }
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index 84510fe21..0ddbe7d1c 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -94,7 +94,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
     }
 
     #[inline]
-    fn dep_kind_info(&self, dep_kind: DepKind) -> &DepKindStruct<'tcx> {
-        &self.query_kinds[dep_kind as usize]
+    fn dep_kind_info(&self, dk: DepKind) -> &DepKindStruct<'tcx> {
+        &self.query_kinds[dk as usize]
     }
 }
diff --git a/compiler/rustc_middle/src/error.rs b/compiler/rustc_middle/src/error.rs
index 5e94da8cb..dc4aa1864 100644
--- a/compiler/rustc_middle/src/error.rs
+++ b/compiler/rustc_middle/src/error.rs
@@ -50,6 +50,14 @@ pub struct LimitInvalid<'a> {
 }
 
 #[derive(Diagnostic)]
+#[diag(middle_recursion_limit_reached)]
+#[help]
+pub struct RecursionLimitReached<'tcx> {
+    pub ty: Ty<'tcx>,
+    pub suggested_limit: rustc_session::Limit,
+}
+
+#[derive(Diagnostic)]
 #[diag(middle_const_eval_non_int)]
 pub struct ConstEvalNonIntError {
     #[primary_span]
diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs
index 4b5bacac8..e551c76f8 100644
--- a/compiler/rustc_middle/src/hir/map/mod.rs
+++ b/compiler/rustc_middle/src/hir/map/mod.rs
@@ -1,13 +1,14 @@
 use crate::hir::{ModuleItems, Owner};
-use crate::ty::{DefIdTree, TyCtxt};
+use crate::query::LocalCrate;
+use crate::ty::TyCtxt;
 use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::{par_for_each_in, Send, Sync};
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
-use rustc_hir::definitions::{DefKey, DefPath, DefPathHash};
+use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
+use rustc_hir::definitions::{DefKey, DefPath, DefPathData, DefPathHash};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::*;
 use rustc_index::vec::Idx;
@@ -73,18 +74,17 @@ impl<'hir> Iterator for ParentHirIterator<'hir> {
         if self.current_id == CRATE_HIR_ID {
             return None;
         }
-        loop {
-            // There are nodes that do not have entries, so we need to skip them.
-            let parent_id = self.map.parent_id(self.current_id);
 
-            if parent_id == self.current_id {
-                self.current_id = CRATE_HIR_ID;
-                return None;
-            }
+        // There are nodes that do not have entries, so we need to skip them.
+        let parent_id = self.map.parent_id(self.current_id);
 
-            self.current_id = parent_id;
-            return Some(parent_id);
+        if parent_id == self.current_id {
+            self.current_id = CRATE_HIR_ID;
+            return None;
         }
+
+        self.current_id = parent_id;
+        return Some(parent_id);
     }
 }
 
@@ -179,7 +179,19 @@ impl<'hir> Map<'hir> {
     /// Do not call this function directly. The query should be called.
     pub(super) fn opt_def_kind(self, local_def_id: LocalDefId) -> Option<DefKind> {
         let hir_id = self.local_def_id_to_hir_id(local_def_id);
-        let def_kind = match self.find(hir_id)? {
+        let node = match self.find(hir_id) {
+            Some(node) => node,
+            None => match self.def_key(local_def_id).disambiguated_data.data {
+                // FIXME: Some anonymous constants do not have corresponding HIR nodes,
+                // so many local queries will panic on their def ids. `None` is currently
+                // returned here instead of `DefKind::{Anon,Inline}Const` to avoid such panics.
+                // Ideally all def ids should have `DefKind`s, we need to create the missing
+                // HIR nodes or feed relevant query results to achieve that.
+                DefPathData::AnonConst => return None,
+                _ => bug!("no HIR node for def id {local_def_id:?}"),
+            },
+        };
+        let def_kind = match node {
             Node::Item(item) => match item.kind {
                 ItemKind::Static(_, mt, _) => DefKind::Static(mt),
                 ItemKind::Const(..) => DefKind::Const,
@@ -187,7 +199,7 @@ impl<'hir> Map<'hir> {
                 ItemKind::Macro(_, macro_kind) => DefKind::Macro(macro_kind),
                 ItemKind::Mod(..) => DefKind::Mod,
                 ItemKind::OpaqueTy(ref opaque) => {
-                    if opaque.in_trait {
+                    if opaque.in_trait && !self.tcx.lower_impl_trait_in_trait_to_assoc_ty() {
                         DefKind::ImplTraitPlaceholder
                     } else {
                         DefKind::OpaqueTy
@@ -266,7 +278,10 @@ impl<'hir> Map<'hir> {
             | Node::Param(_)
             | Node::Arm(_)
             | Node::Lifetime(_)
-            | Node::Block(_) => return None,
+            | Node::Block(_) => span_bug!(
+                self.span(hir_id),
+                "unexpected node with def id {local_def_id:?}: {node:?}"
+            ),
         };
         Some(def_kind)
     }
@@ -316,7 +331,7 @@ impl<'hir> Map<'hir> {
     /// Retrieves the `Node` corresponding to `id`, returning `None` if cannot be found.
     #[inline]
     pub fn find_by_def_id(self, id: LocalDefId) -> Option<Node<'hir>> {
-        self.find(self.local_def_id_to_hir_id(id))
+        self.find(self.tcx.opt_local_def_id_to_hir_id(id)?)
     }
 
     /// Retrieves the `Node` corresponding to `id`, panicking if it cannot be found.
@@ -333,7 +348,7 @@ impl<'hir> Map<'hir> {
     }
 
     pub fn get_if_local(self, id: DefId) -> Option<Node<'hir>> {
-        id.as_local().and_then(|id| self.find(self.local_def_id_to_hir_id(id)))
+        id.as_local().and_then(|id| self.find(self.tcx.opt_local_def_id_to_hir_id(id)?))
     }
 
     pub fn get_generics(self, id: LocalDefId) -> Option<&'hir Generics<'hir>> {
@@ -1131,10 +1146,9 @@ impl<'hir> intravisit::Map<'hir> for Map<'hir> {
     }
 }
 
-pub(super) fn crate_hash(tcx: TyCtxt<'_>, crate_num: CrateNum) -> Svh {
-    debug_assert_eq!(crate_num, LOCAL_CRATE);
+pub(super) fn crate_hash(tcx: TyCtxt<'_>, _: LocalCrate) -> Svh {
     let krate = tcx.hir_crate(());
-    let hir_body_hash = krate.hir_hash;
+    let hir_body_hash = krate.opt_hir_hash.expect("HIR hash missing while computing crate hash");
 
     let upstream_crates = upstream_crates(tcx);
 
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index c9da711e5..7770a5e47 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -7,8 +7,7 @@ pub mod nested_filter;
 pub mod place;
 
 use crate::ty::query::Providers;
-use crate::ty::{DefIdTree, ImplSubject, TyCtxt};
-use rustc_data_structures::fingerprint::Fingerprint;
+use crate::ty::{EarlyBinder, ImplSubject, TyCtxt};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::{par_for_each_in, Send, Sync};
 use rustc_hir::def_id::{DefId, LocalDefId};
@@ -24,14 +23,15 @@ use rustc_span::{ExpnId, DUMMY_SP};
 #[derive(Copy, Clone, Debug)]
 pub struct Owner<'tcx> {
     node: OwnerNode<'tcx>,
-    hash_without_bodies: Fingerprint,
 }
 
 impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for Owner<'tcx> {
     #[inline]
     fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        let Owner { node: _, hash_without_bodies } = self;
-        hash_without_bodies.hash_stable(hcx, hasher)
+        // Perform a shallow hash instead using the deep hash saved in `OwnerNodes`. This lets us
+        // differentiate queries that depend on the full HIR tree from those that only depend on
+        // the item signature.
+        hcx.without_hir_bodies(|hcx| self.node.hash_stable(hcx, hasher));
     }
 }
 
@@ -104,11 +104,11 @@ impl<'tcx> TyCtxt<'tcx> {
         self.parent_module_from_def_id(id.owner.def_id)
     }
 
-    pub fn impl_subject(self, def_id: DefId) -> ImplSubject<'tcx> {
-        self.impl_trait_ref(def_id)
-            .map(|t| t.subst_identity())
-            .map(ImplSubject::Trait)
-            .unwrap_or_else(|| ImplSubject::Inherent(self.type_of(def_id).subst_identity()))
+    pub fn impl_subject(self, def_id: DefId) -> EarlyBinder<ImplSubject<'tcx>> {
+        match self.impl_trait_ref(def_id) {
+            Some(t) => t.map_bound(ImplSubject::Trait),
+            None => self.type_of(def_id).map_bound(ImplSubject::Inherent),
+        }
     }
 }
 
@@ -123,7 +123,7 @@ pub fn provide(providers: &mut Providers) {
     providers.hir_owner = |tcx, id| {
         let owner = tcx.hir_crate(()).owners.get(id.def_id)?.as_owner()?;
         let node = owner.node();
-        Some(Owner { node, hash_without_bodies: owner.nodes.hash_without_bodies })
+        Some(Owner { node })
     };
     providers.opt_local_def_id_to_hir_id = |tcx, id| {
         let owner = tcx.hir_crate(()).owners[id].map(|_| ());
@@ -147,18 +147,18 @@ pub fn provide(providers: &mut Providers) {
         tcx.hir_crate(()).owners[id.def_id].as_owner().map_or(AttributeMap::EMPTY, |o| &o.attrs)
     };
     providers.def_span = |tcx, def_id| {
-        let def_id = def_id.expect_local();
+        let def_id = def_id;
         let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
         tcx.hir().opt_span(hir_id).unwrap_or(DUMMY_SP)
     };
     providers.def_ident_span = |tcx, def_id| {
-        let def_id = def_id.expect_local();
+        let def_id = def_id;
         let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
         tcx.hir().opt_ident_span(hir_id)
     };
     providers.fn_arg_names = |tcx, id| {
         let hir = tcx.hir();
-        let def_id = id.expect_local();
+        let def_id = id;
         let hir_id = hir.local_def_id_to_hir_id(def_id);
         if let Some(body_id) = hir.maybe_body_owned_by(def_id) {
             tcx.arena.alloc_from_iter(hir.body_param_names(body_id))
@@ -176,13 +176,10 @@ pub fn provide(providers: &mut Providers) {
             span_bug!(hir.span(hir_id), "fn_arg_names: unexpected item {:?}", id);
         }
     };
-    providers.opt_def_kind = |tcx, def_id| tcx.hir().opt_def_kind(def_id.expect_local());
-    providers.opt_rpitit_info = |_, _| None;
+    providers.opt_def_kind = |tcx, def_id| tcx.hir().opt_def_kind(def_id);
     providers.all_local_trait_impls = |tcx, ()| &tcx.resolutions(()).trait_impls;
-    providers.expn_that_defined = |tcx, id| {
-        let id = id.expect_local();
-        tcx.resolutions(()).expn_that_defined.get(&id).copied().unwrap_or(ExpnId::root())
-    };
+    providers.expn_that_defined =
+        |tcx, id| tcx.resolutions(()).expn_that_defined.get(&id).copied().unwrap_or(ExpnId::root());
     providers.in_scope_traits_map = |tcx, id| {
         tcx.hir_crate(()).owners[id.def_id].as_owner().map(|owner_info| &owner_info.trait_map)
     };
diff --git a/compiler/rustc_middle/src/hir/place.rs b/compiler/rustc_middle/src/hir/place.rs
index 83d3b0100..80b4c964c 100644
--- a/compiler/rustc_middle/src/hir/place.rs
+++ b/compiler/rustc_middle/src/hir/place.rs
@@ -2,7 +2,7 @@ use crate::ty;
 use crate::ty::Ty;
 
 use rustc_hir::HirId;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, TyEncodable, TyDecodable, HashStable)]
 #[derive(TypeFoldable, TypeVisitable)]
@@ -27,7 +27,7 @@ pub enum ProjectionKind {
     /// the field. The field is identified by which variant
     /// it appears in along with a field index. The variant
     /// is used for enums.
-    Field(u32, VariantIdx),
+    Field(FieldIdx, VariantIdx),
 
     /// Some index like `B[x]`, where `B` is the base
     /// expression. We don't preserve the index `x` because
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index 7f8fc1774..b5b712c36 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -35,9 +35,9 @@ use std::ops::Index;
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TyDecodable, TyEncodable)]
 #[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct Canonical<'tcx, V> {
+    pub value: V,
     pub max_universe: ty::UniverseIndex,
     pub variables: CanonicalVarInfos<'tcx>,
-    pub value: V,
 }
 
 pub type CanonicalVarInfos<'tcx> = &'tcx List<CanonicalVarInfo<'tcx>>;
@@ -80,6 +80,18 @@ impl CanonicalVarValues<'_> {
             }
         })
     }
+
+    pub fn is_identity_modulo_regions(&self) -> bool {
+        self.var_values.iter().enumerate().all(|(bv, arg)| match arg.unpack() {
+            ty::GenericArgKind::Lifetime(_) => true,
+            ty::GenericArgKind::Type(ty) => {
+                matches!(*ty.kind(), ty::Bound(ty::INNERMOST, bt) if bt.var.as_usize() == bv)
+            }
+            ty::GenericArgKind::Const(ct) => {
+                matches!(ct.kind(), ty::ConstKind::Bound(ty::INNERMOST, bc) if bc.as_usize() == bv)
+            }
+        })
+    }
 }
 
 /// When we canonicalize a value to form a query, we wind up replacing
@@ -149,15 +161,15 @@ impl<'tcx> CanonicalVarInfo<'tcx> {
         }
     }
 
-    pub fn expect_anon_placeholder(self) -> u32 {
+    pub fn expect_placeholder_index(self) -> usize {
         match self.kind {
             CanonicalVarKind::Ty(_)
             | CanonicalVarKind::Region(_)
             | CanonicalVarKind::Const(_, _) => bug!("expected placeholder: {self:?}"),
 
-            CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.name.expect_anon(),
-            CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.name.expect_anon(),
-            CanonicalVarKind::PlaceholderConst(placeholder, _) => placeholder.name.as_u32(),
+            CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.bound.var.as_usize(),
+            CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.bound.var.as_usize(),
+            CanonicalVarKind::PlaceholderConst(placeholder, _) => placeholder.bound.as_usize(),
         }
     }
 }
@@ -411,7 +423,7 @@ impl<'tcx> CanonicalVarValues<'tcx> {
                         CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => {
                             let br = ty::BoundRegion {
                                 var: ty::BoundVar::from_usize(i),
-                                kind: ty::BrAnon(i as u32, None),
+                                kind: ty::BrAnon(None),
                             };
                             tcx.mk_re_late_bound(ty::INNERMOST, br).into()
                         }
diff --git a/compiler/rustc_middle/src/infer/unify_key.rs b/compiler/rustc_middle/src/infer/unify_key.rs
index 41d8c7ffd..a873854f0 100644
--- a/compiler/rustc_middle/src/infer/unify_key.rs
+++ b/compiler/rustc_middle/src/infer/unify_key.rs
@@ -1,4 +1,4 @@
-use crate::ty::{self, Ty, TyCtxt};
+use crate::ty::{self, Region, Ty, TyCtxt};
 use rustc_data_structures::unify::{NoError, UnifyKey, UnifyValue};
 use rustc_span::def_id::DefId;
 use rustc_span::symbol::Symbol;
@@ -11,7 +11,20 @@ pub trait ToType {
 }
 
 #[derive(PartialEq, Copy, Clone, Debug)]
-pub struct UnifiedRegion<'tcx>(pub Option<ty::Region<'tcx>>);
+pub struct UnifiedRegion<'tcx> {
+    value: Option<ty::Region<'tcx>>,
+}
+
+impl<'tcx> UnifiedRegion<'tcx> {
+    pub fn new(value: Option<Region<'tcx>>) -> Self {
+        Self { value }
+    }
+
+    /// The caller is responsible for checking universe compatibility before using this value.
+    pub fn get_value_ignoring_universes(self) -> Option<Region<'tcx>> {
+        self.value
+    }
+}
 
 #[derive(PartialEq, Copy, Clone, Debug)]
 pub struct RegionVidKey<'tcx> {
@@ -44,11 +57,27 @@ impl<'tcx> UnifyValue for UnifiedRegion<'tcx> {
     type Error = NoError;
 
     fn unify_values(value1: &Self, value2: &Self) -> Result<Self, NoError> {
-        Ok(match (value1.0, value2.0) {
+        // We pick the value of the least universe because it is compatible with more variables.
+        // This is *not* necessary for soundness, but it allows more region variables to be
+        // resolved to the said value.
+        #[cold]
+        fn min_universe<'tcx>(r1: Region<'tcx>, r2: Region<'tcx>) -> Region<'tcx> {
+            cmp::min_by_key(r1, r2, |r| match r.kind() {
+                ty::ReStatic
+                | ty::ReErased
+                | ty::ReFree(..)
+                | ty::ReEarlyBound(..)
+                | ty::ReError(_) => ty::UniverseIndex::ROOT,
+                ty::RePlaceholder(placeholder) => placeholder.universe,
+                ty::ReVar(..) | ty::ReLateBound(..) => bug!("not a universal region"),
+            })
+        }
+
+        Ok(match (value1.value, value2.value) {
             // Here we can just pick one value, because the full constraints graph
             // will be handled later. Ideally, we might want a `MultipleValues`
             // variant or something. For now though, this is fine.
-            (Some(_), Some(_)) => *value1,
+            (Some(val1), Some(val2)) => Self { value: Some(min_universe(val1, val2)) },
 
             (Some(_), _) => *value1,
             (_, Some(_)) => *value2,
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index c33b9d84e..b4edb02f6 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -33,13 +33,13 @@
 #![feature(generators)]
 #![feature(get_mut_unchecked)]
 #![feature(if_let_guard)]
+#![feature(inline_const)]
 #![feature(iter_from_generator)]
 #![feature(local_key_cell_methods)]
 #![feature(negative_impls)]
 #![feature(never_type)]
 #![feature(extern_types)]
 #![feature(new_uninit)]
-#![feature(once_cell)]
 #![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(trusted_len)]
@@ -109,4 +109,4 @@ pub mod util {
 // Allows macros to refer to this crate as `::rustc_middle`
 extern crate self as rustc_middle;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_middle/src/macros.rs b/compiler/rustc_middle/src/macros.rs
index a8d71ce03..89014f62d 100644
--- a/compiler/rustc_middle/src/macros.rs
+++ b/compiler/rustc_middle/src/macros.rs
@@ -1,6 +1,6 @@
 /// A macro for triggering an ICE.
 /// Calling `bug` instead of panicking will result in a nicer error message and should
-/// therefore be prefered over `panic`/`unreachable` or others.
+/// therefore be preferred over `panic`/`unreachable` or others.
 ///
 /// If you have a span available, you should use [`span_bug`] instead.
 ///
diff --git a/compiler/rustc_middle/src/metadata.rs b/compiler/rustc_middle/src/metadata.rs
index 5ff014c78..f3170e0ec 100644
--- a/compiler/rustc_middle/src/metadata.rs
+++ b/compiler/rustc_middle/src/metadata.rs
@@ -5,13 +5,34 @@ use rustc_macros::HashStable;
 use rustc_span::def_id::DefId;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
+use smallvec::SmallVec;
+
+/// A simplified version of `ImportKind` from resolve.
+/// `DefId`s here correspond to `use` and `extern crate` items themselves, not their targets.
+#[derive(Clone, Copy, Debug, TyEncodable, TyDecodable, HashStable)]
+pub enum Reexport {
+    Single(DefId),
+    Glob(DefId),
+    ExternCrate(DefId),
+    MacroUse,
+    MacroExport,
+}
+
+impl Reexport {
+    pub fn id(self) -> Option<DefId> {
+        match self {
+            Reexport::Single(id) | Reexport::Glob(id) | Reexport::ExternCrate(id) => Some(id),
+            Reexport::MacroUse | Reexport::MacroExport => None,
+        }
+    }
+}
 
 /// This structure is supposed to keep enough data to re-create `NameBinding`s for other crates
 /// during name resolution. Right now the bindings are not recreated entirely precisely so we may
 /// need to add more data in the future to correctly support macros 2.0, for example.
 /// Module child can be either a proper item or a reexport (including private imports).
 /// In case of reexport all the fields describe the reexport item itself, not what it refers to.
-#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
+#[derive(Debug, TyEncodable, TyDecodable, HashStable)]
 pub struct ModChild {
     /// Name of the item.
     pub ident: Ident,
@@ -22,6 +43,7 @@ pub struct ModChild {
     pub vis: ty::Visibility<DefId>,
     /// Span of the item.
     pub span: Span,
-    /// A proper `macro_rules` item (not a reexport).
-    pub macro_rules: bool,
+    /// Reexport chain linking this module child to its original reexported item.
+    /// Empty if the module child is a proper item.
+    pub reexport_chain: SmallVec<[Reexport; 2]>,
 }
diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs
index 631fd09ec..c0c0fd07b 100644
--- a/compiler/rustc_middle/src/middle/exported_symbols.rs
+++ b/compiler/rustc_middle/src/middle/exported_symbols.rs
@@ -43,6 +43,7 @@ pub enum ExportedSymbol<'tcx> {
     NonGeneric(DefId),
     Generic(DefId, SubstsRef<'tcx>),
     DropGlue(Ty<'tcx>),
+    ThreadLocalShim(DefId),
     NoDefId(ty::SymbolName<'tcx>),
 }
 
@@ -58,6 +59,10 @@ impl<'tcx> ExportedSymbol<'tcx> {
             ExportedSymbol::DropGlue(ty) => {
                 tcx.symbol_name(ty::Instance::resolve_drop_in_place(tcx, ty))
             }
+            ExportedSymbol::ThreadLocalShim(def_id) => tcx.symbol_name(ty::Instance {
+                def: ty::InstanceDef::ThreadLocalShim(def_id),
+                substs: ty::InternalSubsts::empty(),
+            }),
             ExportedSymbol::NoDefId(symbol_name) => symbol_name,
         }
     }
diff --git a/compiler/rustc_middle/src/middle/mod.rs b/compiler/rustc_middle/src/middle/mod.rs
index 0b6774f1b..9c25f3009 100644
--- a/compiler/rustc_middle/src/middle/mod.rs
+++ b/compiler/rustc_middle/src/middle/mod.rs
@@ -19,7 +19,7 @@ pub mod lib_features {
                 .stable
                 .iter()
                 .map(|(f, (s, _))| (*f, Some(*s)))
-                .chain(self.unstable.iter().map(|(f, _)| (*f, None)))
+                .chain(self.unstable.keys().map(|f| (*f, None)))
                 .collect();
             all_features.sort_unstable_by(|a, b| a.0.as_str().partial_cmp(b.0.as_str()).unwrap());
             all_features
diff --git a/compiler/rustc_middle/src/middle/privacy.rs b/compiler/rustc_middle/src/middle/privacy.rs
index 893bf54b8..967fed687 100644
--- a/compiler/rustc_middle/src/middle/privacy.rs
+++ b/compiler/rustc_middle/src/middle/privacy.rs
@@ -1,12 +1,12 @@
 //! A pass that checks to make sure private fields and methods aren't used
 //! outside their scopes. This pass will also generate a set of exported items
 //! which are available for use externally when compiled as a library.
-use crate::ty::{DefIdTree, TyCtxt, Visibility};
+use crate::ty::{TyCtxt, Visibility};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_macros::HashStable;
 use rustc_query_system::ich::StableHashingContext;
-use rustc_span::def_id::LocalDefId;
+use rustc_span::def_id::{LocalDefId, CRATE_DEF_ID};
 use std::hash::Hash;
 
 /// Represents the levels of effective visibility an item can have.
@@ -107,12 +107,16 @@ impl EffectiveVisibilities {
         })
     }
 
+    pub fn update_root(&mut self) {
+        self.map.insert(CRATE_DEF_ID, EffectiveVisibility::from_vis(Visibility::Public));
+    }
+
     // FIXME: Share code with `fn update`.
     pub fn update_eff_vis(
         &mut self,
         def_id: LocalDefId,
         eff_vis: &EffectiveVisibility,
-        tree: impl DefIdTree,
+        tcx: TyCtxt<'_>,
     ) {
         use std::collections::hash_map::Entry;
         match self.map.entry(def_id) {
@@ -122,7 +126,7 @@ impl EffectiveVisibilities {
                     let vis_at_level = eff_vis.at_level(l);
                     let old_vis_at_level = old_eff_vis.at_level_mut(l);
                     if vis_at_level != old_vis_at_level
-                        && vis_at_level.is_at_least(*old_vis_at_level, tree)
+                        && vis_at_level.is_at_least(*old_vis_at_level, tcx)
                     {
                         *old_vis_at_level = *vis_at_level
                     }
@@ -219,7 +223,7 @@ impl<Id: Eq + Hash> EffectiveVisibilities<Id> {
         lazy_private_vis: impl FnOnce() -> Visibility,
         inherited_effective_vis: EffectiveVisibility,
         level: Level,
-        tree: impl DefIdTree,
+        tcx: TyCtxt<'_>,
     ) -> bool {
         let mut changed = false;
         let mut current_effective_vis = self
@@ -240,7 +244,7 @@ impl<Id: Eq + Hash> EffectiveVisibilities<Id> {
                     && level != l)
                 {
                     calculated_effective_vis =
-                        if nominal_vis.is_at_least(inherited_effective_vis_at_level, tree) {
+                        if nominal_vis.is_at_least(inherited_effective_vis_at_level, tcx) {
                             inherited_effective_vis_at_level
                         } else {
                             nominal_vis
@@ -249,7 +253,7 @@ impl<Id: Eq + Hash> EffectiveVisibilities<Id> {
                 // effective visibility can't be decreased at next update call for the
                 // same id
                 if *current_effective_vis_at_level != calculated_effective_vis
-                    && calculated_effective_vis.is_at_least(*current_effective_vis_at_level, tree)
+                    && calculated_effective_vis.is_at_least(*current_effective_vis_at_level, tcx)
                 {
                     changed = true;
                     *current_effective_vis_at_level = calculated_effective_vis;
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index 354c84e22..b61f7806b 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -3,7 +3,7 @@
 
 pub use self::StabilityLevel::*;
 
-use crate::ty::{self, DefIdTree, TyCtxt};
+use crate::ty::{self, TyCtxt};
 use rustc_ast::NodeId;
 use rustc_attr::{self as attr, ConstStability, DefaultBodyStability, Deprecation, Stability};
 use rustc_data_structures::fx::FxHashMap;
diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs
index b93871769..3fb468379 100644
--- a/compiler/rustc_middle/src/mir/basic_blocks.rs
+++ b/compiler/rustc_middle/src/mir/basic_blocks.rs
@@ -6,7 +6,7 @@ use rustc_data_structures::graph;
 use rustc_data_structures::graph::dominators::{dominators, Dominators};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::OnceCell;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use smallvec::SmallVec;
 
@@ -124,10 +124,10 @@ impl<'tcx> BasicBlocks<'tcx> {
 }
 
 impl<'tcx> std::ops::Deref for BasicBlocks<'tcx> {
-    type Target = IndexVec<BasicBlock, BasicBlockData<'tcx>>;
+    type Target = IndexSlice<BasicBlock, BasicBlockData<'tcx>>;
 
     #[inline]
-    fn deref(&self) -> &IndexVec<BasicBlock, BasicBlockData<'tcx>> {
+    fn deref(&self) -> &IndexSlice<BasicBlock, BasicBlockData<'tcx>> {
         &self.basic_blocks
     }
 }
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index 48375ed30..1a8e48264 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -2,8 +2,6 @@
 
 mod init_mask;
 mod provenance_map;
-#[cfg(test)]
-mod tests;
 
 use std::borrow::Cow;
 use std::fmt;
@@ -111,26 +109,34 @@ const MAX_HASHED_BUFFER_LEN: usize = 2 * MAX_BYTES_TO_HASH;
 // large.
 impl hash::Hash for Allocation {
     fn hash<H: hash::Hasher>(&self, state: &mut H) {
+        let Self {
+            bytes,
+            provenance,
+            init_mask,
+            align,
+            mutability,
+            extra: (), // don't bother hashing ()
+        } = self;
+
         // Partially hash the `bytes` buffer when it is large. To limit collisions with common
         // prefixes and suffixes, we hash the length and some slices of the buffer.
-        let byte_count = self.bytes.len();
+        let byte_count = bytes.len();
         if byte_count > MAX_HASHED_BUFFER_LEN {
             // Hash the buffer's length.
             byte_count.hash(state);
 
             // And its head and tail.
-            self.bytes[..MAX_BYTES_TO_HASH].hash(state);
-            self.bytes[byte_count - MAX_BYTES_TO_HASH..].hash(state);
+            bytes[..MAX_BYTES_TO_HASH].hash(state);
+            bytes[byte_count - MAX_BYTES_TO_HASH..].hash(state);
         } else {
-            self.bytes.hash(state);
+            bytes.hash(state);
         }
 
         // Hash the other fields as usual.
-        self.provenance.hash(state);
-        self.init_mask.hash(state);
-        self.align.hash(state);
-        self.mutability.hash(state);
-        self.extra.hash(state);
+        provenance.hash(state);
+        init_mask.hash(state);
+        align.hash(state);
+        mutability.hash(state);
     }
 }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs b/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs
index 82e9a961a..dcb56a175 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs
@@ -1,3 +1,6 @@
+#[cfg(test)]
+mod tests;
+
 use std::hash;
 use std::iter;
 use std::ops::Range;
@@ -10,20 +13,185 @@ type Block = u64;
 
 /// A bitmask where each bit refers to the byte with the same index. If the bit is `true`, the byte
 /// is initialized. If it is `false` the byte is uninitialized.
-// Note: for performance reasons when interning, some of the `InitMask` fields can be partially
-// hashed. (see the `Hash` impl below for more details), so the impl is not derived.
-#[derive(Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable)]
-#[derive(HashStable)]
+/// The actual bits are only materialized when needed, and we try to keep this data lazy as long as
+/// possible. Currently, if all the blocks have the same value, then the mask represents either a
+/// fully initialized or fully uninitialized const allocation, so we can only store that single
+/// value.
+#[derive(Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)]
 pub struct InitMask {
-    blocks: Vec<Block>,
+    blocks: InitMaskBlocks,
     len: Size,
 }
 
+#[derive(Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)]
+enum InitMaskBlocks {
+    Lazy {
+        /// Whether the lazy init mask is fully initialized or uninitialized.
+        state: bool,
+    },
+    Materialized(InitMaskMaterialized),
+}
+
+impl InitMask {
+    pub fn new(size: Size, state: bool) -> Self {
+        // Blocks start lazily allocated, until we have to materialize them.
+        let blocks = InitMaskBlocks::Lazy { state };
+        InitMask { len: size, blocks }
+    }
+
+    /// Checks whether the `range` is entirely initialized.
+    ///
+    /// Returns `Ok(())` if it's initialized. Otherwise returns a range of byte
+    /// indexes for the first contiguous span of the uninitialized access.
+    #[inline]
+    pub fn is_range_initialized(&self, range: AllocRange) -> Result<(), AllocRange> {
+        let end = range.end();
+        if end > self.len {
+            return Err(AllocRange::from(self.len..end));
+        }
+
+        match self.blocks {
+            InitMaskBlocks::Lazy { state } => {
+                // Lazily allocated blocks represent the full mask, and cover the requested range by
+                // definition.
+                if state { Ok(()) } else { Err(range) }
+            }
+            InitMaskBlocks::Materialized(ref blocks) => {
+                blocks.is_range_initialized(range.start, end)
+            }
+        }
+    }
+
+    /// Sets a specified range to a value. If the range is out-of-bounds, the mask will grow to
+    /// accommodate it entirely.
+    pub fn set_range(&mut self, range: AllocRange, new_state: bool) {
+        let start = range.start;
+        let end = range.end();
+
+        let is_full_overwrite = start == Size::ZERO && end >= self.len;
+
+        // Optimize the cases of a full init/uninit state, while handling growth if needed.
+        match self.blocks {
+            InitMaskBlocks::Lazy { ref mut state } if is_full_overwrite => {
+                // This is fully overwriting the mask, and we'll still have a single initialization
+                // state: the blocks can stay lazy.
+                *state = new_state;
+                self.len = end;
+            }
+            InitMaskBlocks::Materialized(_) if is_full_overwrite => {
+                // This is also fully overwriting materialized blocks with a single initialization
+                // state: we'll have no need for these blocks anymore and can make them lazy.
+                self.blocks = InitMaskBlocks::Lazy { state: new_state };
+                self.len = end;
+            }
+            InitMaskBlocks::Lazy { state } if state == new_state => {
+                // Here we're partially overwriting the mask but the initialization state doesn't
+                // change: the blocks can stay lazy.
+                if end > self.len {
+                    self.len = end;
+                }
+            }
+            _ => {
+                // Otherwise, we have a partial overwrite that can result in a mix of initialization
+                // states, so we'll need materialized blocks.
+                let len = self.len;
+                let blocks = self.materialize_blocks();
+
+                // There are 3 cases of interest here, if we have:
+                //
+                //         [--------]
+                //         ^        ^
+                //         0        len
+                //
+                // 1) the range to set can be in-bounds:
+                //
+                //            xxxx = [start, end]
+                //         [--------]
+                //         ^        ^
+                //         0        len
+                //
+                // Here, we'll simply set the single `start` to `end` range.
+                //
+                // 2) the range to set can be partially out-of-bounds:
+                //
+                //                xxxx = [start, end]
+                //         [--------]
+                //         ^        ^
+                //         0        len
+                //
+                // We have 2 subranges to handle:
+                // - we'll set the existing `start` to `len` range.
+                // - we'll grow and set the `len` to `end` range.
+                //
+                // 3) the range to set can be fully out-of-bounds:
+                //
+                //                   ---xxxx = [start, end]
+                //         [--------]
+                //         ^        ^
+                //         0        len
+                //
+                // Since we're growing the mask to a single `new_state` value, we consider the gap
+                // from `len` to `start` to be part of the range, and have a single subrange to
+                // handle: we'll grow and set the `len` to `end` range.
+                //
+                // Note that we have to materialize, set blocks, and grow the mask. We could
+                // therefore slightly optimize things in situations where these writes overlap.
+                // However, as of writing this, growing the mask doesn't happen in practice yet, so
+                // we don't do this micro-optimization.
+
+                if end <= len {
+                    // Handle case 1.
+                    blocks.set_range_inbounds(start, end, new_state);
+                } else {
+                    if start < len {
+                        // Handle the first subrange of case 2.
+                        blocks.set_range_inbounds(start, len, new_state);
+                    }
+
+                    // Handle the second subrange of case 2, and case 3.
+                    blocks.grow(len, end - len, new_state); // `Size` operation
+                    self.len = end;
+                }
+            }
+        }
+    }
+
+    /// Materializes this mask's blocks when the mask is lazy.
+    #[inline]
+    fn materialize_blocks(&mut self) -> &mut InitMaskMaterialized {
+        if let InitMaskBlocks::Lazy { state } = self.blocks {
+            self.blocks = InitMaskBlocks::Materialized(InitMaskMaterialized::new(self.len, state));
+        }
+
+        let InitMaskBlocks::Materialized(ref mut blocks) = self.blocks else {
+            bug!("initmask blocks must be materialized here")
+        };
+        blocks
+    }
+
+    /// Returns the initialization state at the specified in-bounds index.
+    #[inline]
+    pub fn get(&self, idx: Size) -> bool {
+        match self.blocks {
+            InitMaskBlocks::Lazy { state } => state,
+            InitMaskBlocks::Materialized(ref blocks) => blocks.get(idx),
+        }
+    }
+}
+
+/// The actual materialized blocks of the bitmask, when we can't keep the `InitMask` lazy.
+// Note: for performance reasons when interning, some of the fields can be partially
+// hashed. (see the `Hash` impl below for more details), so the impl is not derived.
+#[derive(Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, HashStable)]
+struct InitMaskMaterialized {
+    blocks: Vec<Block>,
+}
+
 // Const allocations are only hashed for interning. However, they can be large, making the hashing
 // expensive especially since it uses `FxHash`: it's better suited to short keys, not potentially
 // big buffers like the allocation's init mask. We can partially hash some fields when they're
 // large.
-impl hash::Hash for InitMask {
+impl hash::Hash for InitMaskMaterialized {
     fn hash<H: hash::Hasher>(&self, state: &mut H) {
         const MAX_BLOCKS_TO_HASH: usize = super::MAX_BYTES_TO_HASH / std::mem::size_of::<Block>();
         const MAX_BLOCKS_LEN: usize = super::MAX_HASHED_BUFFER_LEN / std::mem::size_of::<Block>();
@@ -41,18 +209,15 @@ impl hash::Hash for InitMask {
         } else {
             self.blocks.hash(state);
         }
-
-        // Hash the other fields as usual.
-        self.len.hash(state);
     }
 }
 
-impl InitMask {
+impl InitMaskMaterialized {
     pub const BLOCK_SIZE: u64 = 64;
 
-    pub fn new(size: Size, state: bool) -> Self {
-        let mut m = InitMask { blocks: vec![], len: Size::ZERO };
-        m.grow(size, state);
+    fn new(size: Size, state: bool) -> Self {
+        let mut m = InitMaskMaterialized { blocks: vec![] };
+        m.grow(Size::ZERO, size, state);
         m
     }
 
@@ -62,8 +227,8 @@ impl InitMask {
         // Each bit in a `Block` represents the initialization state of one byte of an allocation,
         // so we use `.bytes()` here.
         let bits = bits.bytes();
-        let a = bits / InitMask::BLOCK_SIZE;
-        let b = bits % InitMask::BLOCK_SIZE;
+        let a = bits / Self::BLOCK_SIZE;
+        let b = bits % Self::BLOCK_SIZE;
         (usize::try_from(a).unwrap(), usize::try_from(b).unwrap())
     }
 
@@ -71,7 +236,7 @@ impl InitMask {
     fn size_from_bit_index(block: impl TryInto<u64>, bit: impl TryInto<u64>) -> Size {
         let block = block.try_into().ok().unwrap();
         let bit = bit.try_into().ok().unwrap();
-        Size::from_bytes(block * InitMask::BLOCK_SIZE + bit)
+        Size::from_bytes(block * Self::BLOCK_SIZE + bit)
     }
 
     /// Checks whether the `range` is entirely initialized.
@@ -79,13 +244,8 @@ impl InitMask {
     /// Returns `Ok(())` if it's initialized. Otherwise returns a range of byte
     /// indexes for the first contiguous span of the uninitialized access.
     #[inline]
-    pub fn is_range_initialized(&self, range: AllocRange) -> Result<(), AllocRange> {
-        let end = range.end();
-        if end > self.len {
-            return Err(AllocRange::from(self.len..end));
-        }
-
-        let uninit_start = self.find_bit(range.start, end, false);
+    fn is_range_initialized(&self, start: Size, end: Size) -> Result<(), AllocRange> {
+        let uninit_start = self.find_bit(start, end, false);
 
         match uninit_start {
             Some(uninit_start) => {
@@ -96,81 +256,80 @@ impl InitMask {
         }
     }
 
-    pub fn set_range(&mut self, range: AllocRange, new_state: bool) {
-        let end = range.end();
-        let len = self.len;
-        if end > len {
-            self.grow(end - len, new_state);
-        }
-        self.set_range_inbounds(range.start, end, new_state);
-    }
-
     fn set_range_inbounds(&mut self, start: Size, end: Size, new_state: bool) {
-        let (blocka, bita) = Self::bit_index(start);
-        let (blockb, bitb) = Self::bit_index(end);
-        if blocka == blockb {
-            // First set all bits except the first `bita`,
-            // then unset the last `64 - bitb` bits.
-            let range = if bitb == 0 {
-                u64::MAX << bita
+        let (block_a, bit_a) = Self::bit_index(start);
+        let (block_b, bit_b) = Self::bit_index(end);
+        if block_a == block_b {
+            // First set all bits except the first `bit_a`,
+            // then unset the last `64 - bit_b` bits.
+            let range = if bit_b == 0 {
+                u64::MAX << bit_a
             } else {
-                (u64::MAX << bita) & (u64::MAX >> (64 - bitb))
+                (u64::MAX << bit_a) & (u64::MAX >> (64 - bit_b))
             };
             if new_state {
-                self.blocks[blocka] |= range;
+                self.blocks[block_a] |= range;
             } else {
-                self.blocks[blocka] &= !range;
+                self.blocks[block_a] &= !range;
             }
             return;
         }
         // across block boundaries
         if new_state {
-            // Set `bita..64` to `1`.
-            self.blocks[blocka] |= u64::MAX << bita;
-            // Set `0..bitb` to `1`.
-            if bitb != 0 {
-                self.blocks[blockb] |= u64::MAX >> (64 - bitb);
+            // Set `bit_a..64` to `1`.
+            self.blocks[block_a] |= u64::MAX << bit_a;
+            // Set `0..bit_b` to `1`.
+            if bit_b != 0 {
+                self.blocks[block_b] |= u64::MAX >> (64 - bit_b);
             }
             // Fill in all the other blocks (much faster than one bit at a time).
-            for block in (blocka + 1)..blockb {
+            for block in (block_a + 1)..block_b {
                 self.blocks[block] = u64::MAX;
             }
         } else {
-            // Set `bita..64` to `0`.
-            self.blocks[blocka] &= !(u64::MAX << bita);
-            // Set `0..bitb` to `0`.
-            if bitb != 0 {
-                self.blocks[blockb] &= !(u64::MAX >> (64 - bitb));
+            // Set `bit_a..64` to `0`.
+            self.blocks[block_a] &= !(u64::MAX << bit_a);
+            // Set `0..bit_b` to `0`.
+            if bit_b != 0 {
+                self.blocks[block_b] &= !(u64::MAX >> (64 - bit_b));
             }
             // Fill in all the other blocks (much faster than one bit at a time).
-            for block in (blocka + 1)..blockb {
+            for block in (block_a + 1)..block_b {
                 self.blocks[block] = 0;
             }
         }
     }
 
     #[inline]
-    pub fn get(&self, i: Size) -> bool {
+    fn get(&self, i: Size) -> bool {
         let (block, bit) = Self::bit_index(i);
         (self.blocks[block] & (1 << bit)) != 0
     }
 
-    fn grow(&mut self, amount: Size, new_state: bool) {
+    fn grow(&mut self, len: Size, amount: Size, new_state: bool) {
         if amount.bytes() == 0 {
             return;
         }
         let unused_trailing_bits =
-            u64::try_from(self.blocks.len()).unwrap() * Self::BLOCK_SIZE - self.len.bytes();
+            u64::try_from(self.blocks.len()).unwrap() * Self::BLOCK_SIZE - len.bytes();
+
+        // If there's not enough capacity in the currently allocated blocks, allocate some more.
         if amount.bytes() > unused_trailing_bits {
             let additional_blocks = amount.bytes() / Self::BLOCK_SIZE + 1;
-            self.blocks.extend(
-                // FIXME(oli-obk): optimize this by repeating `new_state as Block`.
-                iter::repeat(0).take(usize::try_from(additional_blocks).unwrap()),
-            );
+
+            // We allocate the blocks to the correct value for the requested init state, so we won't
+            // have to manually set them with another write.
+            let block = if new_state { u64::MAX } else { 0 };
+            self.blocks
+                .extend(iter::repeat(block).take(usize::try_from(additional_blocks).unwrap()));
+        }
+
+        // New blocks have already been set here, so we only need to set the unused trailing bits,
+        // if any.
+        if unused_trailing_bits > 0 {
+            let in_bounds_tail = Size::from_bytes(unused_trailing_bits);
+            self.set_range_inbounds(len, len + in_bounds_tail, new_state); // `Size` operation
         }
-        let start = self.len;
-        self.len += amount;
-        self.set_range_inbounds(start, start + amount, new_state); // `Size` operation
     }
 
     /// Returns the index of the first bit in `start..end` (end-exclusive) that is equal to is_init.
@@ -188,7 +347,7 @@ impl InitMask {
         /// ```
         /// Also, if not stated, assume that `is_init = true`, that is, we are searching for the first 1 bit.
         fn find_bit_fast(
-            init_mask: &InitMask,
+            init_mask: &InitMaskMaterialized,
             start: Size,
             end: Size,
             is_init: bool,
@@ -223,7 +382,7 @@ impl InitMask {
                     None
                 } else {
                     let bit = bits.trailing_zeros();
-                    Some(InitMask::size_from_bit_index(block, bit))
+                    Some(InitMaskMaterialized::size_from_bit_index(block, bit))
                 }
             }
 
@@ -253,9 +412,9 @@ impl InitMask {
             // This provides the desired behavior of searching blocks 0 and 1 for (a),
             // and searching only block 0 for (b).
             // There is no concern of overflows since we checked for `start >= end` above.
-            let (start_block, start_bit) = InitMask::bit_index(start);
+            let (start_block, start_bit) = InitMaskMaterialized::bit_index(start);
             let end_inclusive = Size::from_bytes(end.bytes() - 1);
-            let (end_block_inclusive, _) = InitMask::bit_index(end_inclusive);
+            let (end_block_inclusive, _) = InitMaskMaterialized::bit_index(end_inclusive);
 
             // Handle first block: need to skip `start_bit` bits.
             //
@@ -340,7 +499,7 @@ impl InitMask {
 
         #[cfg_attr(not(debug_assertions), allow(dead_code))]
         fn find_bit_slow(
-            init_mask: &InitMask,
+            init_mask: &InitMaskMaterialized,
             start: Size,
             end: Size,
             is_init: bool,
@@ -436,10 +595,19 @@ impl<'a> Iterator for InitChunkIter<'a> {
             return None;
         }
 
-        let end_of_chunk =
-            self.init_mask.find_bit(self.start, self.end, !self.is_init).unwrap_or(self.end);
+        let end_of_chunk = match self.init_mask.blocks {
+            InitMaskBlocks::Lazy { .. } => {
+                // If we're iterating over the chunks of lazy blocks, we just emit a single
+                // full-size chunk.
+                self.end
+            }
+            InitMaskBlocks::Materialized(ref blocks) => {
+                let end_of_chunk =
+                    blocks.find_bit(self.start, self.end, !self.is_init).unwrap_or(self.end);
+                end_of_chunk
+            }
+        };
         let range = self.start..end_of_chunk;
-
         let ret =
             Some(if self.is_init { InitChunk::Init(range) } else { InitChunk::Uninit(range) });
 
@@ -504,17 +672,19 @@ impl InitMask {
 
     /// Applies multiple instances of the run-length encoding to the initialization mask.
     pub fn apply_copy(&mut self, defined: InitCopy, range: AllocRange, repeat: u64) {
-        // An optimization where we can just overwrite an entire range of initialization
-        // bits if they are going to be uniformly `1` or `0`.
+        // An optimization where we can just overwrite an entire range of initialization bits if
+        // they are going to be uniformly `1` or `0`. If this happens to be a full-range overwrite,
+        // we won't need materialized blocks either.
         if defined.ranges.len() <= 1 {
-            self.set_range_inbounds(
-                range.start,
-                range.start + range.size * repeat, // `Size` operations
-                defined.initial,
-            );
+            let start = range.start;
+            let end = range.start + range.size * repeat; // `Size` operations
+            self.set_range(AllocRange::from(start..end), defined.initial);
             return;
         }
 
+        // We're about to do one or more partial writes, so we ensure the blocks are materialized.
+        let blocks = self.materialize_blocks();
+
         for mut j in 0..repeat {
             j *= range.size.bytes();
             j += range.start.bytes();
@@ -522,7 +692,7 @@ impl InitMask {
             for range in &defined.ranges {
                 let old_j = j;
                 j += range;
-                self.set_range_inbounds(Size::from_bytes(old_j), Size::from_bytes(j), cur);
+                blocks.set_range_inbounds(Size::from_bytes(old_j), Size::from_bytes(j), cur);
                 cur = !cur;
             }
         }
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/init_mask/tests.rs b/compiler/rustc_middle/src/mir/interpret/allocation/init_mask/tests.rs
new file mode 100644
index 000000000..1a7934bc2
--- /dev/null
+++ b/compiler/rustc_middle/src/mir/interpret/allocation/init_mask/tests.rs
@@ -0,0 +1,195 @@
+use super::*;
+use crate::mir::interpret::alloc_range;
+
+#[test]
+fn uninit_mask() {
+    let mut mask = InitMask::new(Size::from_bytes(500), false);
+    assert!(!mask.get(Size::from_bytes(499)));
+    mask.set_range(alloc_range(Size::from_bytes(499), Size::from_bytes(1)), true);
+    assert!(mask.get(Size::from_bytes(499)));
+    mask.set_range((100..256).into(), true);
+    for i in 0..100 {
+        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
+    }
+    for i in 100..256 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+    for i in 256..499 {
+        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
+    }
+}
+
+/// Returns the number of materialized blocks for this mask.
+fn materialized_block_count(mask: &InitMask) -> usize {
+    match mask.blocks {
+        InitMaskBlocks::Lazy { .. } => 0,
+        InitMaskBlocks::Materialized(ref blocks) => blocks.blocks.len(),
+    }
+}
+
+#[test]
+fn materialize_mask_within_range() {
+    // To have spare bits, we use a mask size smaller than its block size of 64.
+    let mut mask = InitMask::new(Size::from_bytes(16), false);
+    assert_eq!(materialized_block_count(&mask), 0);
+
+    // Forces materialization, but doesn't require growth. This is case #1 documented in the
+    // `set_range` method.
+    mask.set_range((8..16).into(), true);
+    assert_eq!(materialized_block_count(&mask), 1);
+
+    for i in 0..8 {
+        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
+    }
+    for i in 8..16 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+}
+
+#[test]
+fn grow_within_unused_bits_with_full_overwrite() {
+    // To have spare bits, we use a mask size smaller than its block size of 64.
+    let mut mask = InitMask::new(Size::from_bytes(16), true);
+    for i in 0..16 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+
+    // Grow without requiring an additional block. Full overwrite.
+    // This can be fully handled without materialization.
+    let range = (0..32).into();
+    mask.set_range(range, true);
+
+    for i in 0..32 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+
+    assert_eq!(materialized_block_count(&mask), 0);
+}
+
+// This test checks that an initmask's spare capacity is correctly used when growing within block
+// capacity. This can be fully handled without materialization.
+#[test]
+fn grow_same_state_within_unused_bits() {
+    // To have spare bits, we use a mask size smaller than its block size of 64.
+    let mut mask = InitMask::new(Size::from_bytes(16), true);
+    for i in 0..16 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+
+    // Grow without requiring an additional block. The gap between the current length and the
+    // range's beginning should be set to the same value as the range.
+    let range = (24..32).into();
+    mask.set_range(range, true);
+
+    // We want to make sure the unused bits in the first block are correct
+    for i in 16..24 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+
+    for i in 24..32 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+
+    assert_eq!(1, mask.range_as_init_chunks((0..32).into()).count());
+    assert_eq!(materialized_block_count(&mask), 0);
+}
+
+// This is the same test as `grow_same_state_within_unused_bits` but with both init and uninit
+// states: this forces materialization; otherwise the mask could stay lazy even when needing to
+// grow.
+#[test]
+fn grow_mixed_state_within_unused_bits() {
+    // To have spare bits, we use a mask size smaller than its block size of 64.
+    let mut mask = InitMask::new(Size::from_bytes(16), true);
+    for i in 0..16 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+
+    // Grow without requiring an additional block. The gap between the current length and the
+    // range's beginning should be set to the same value as the range. Note: since this is fully
+    // out-of-bounds of the current mask, this is case #3 described in the `set_range` method.
+    let range = (24..32).into();
+    mask.set_range(range, false);
+
+    // We want to make sure the unused bits in the first block are correct
+    for i in 16..24 {
+        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
+    }
+
+    for i in 24..32 {
+        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
+    }
+
+    assert_eq!(1, mask.range_as_init_chunks((0..16).into()).count());
+    assert_eq!(2, mask.range_as_init_chunks((0..32).into()).count());
+    assert_eq!(materialized_block_count(&mask), 1);
+}
+
+// This is similar to `grow_mixed_state_within_unused_bits` to force materialization, but the range
+// to set partially overlaps the mask, so this requires a different growth + write pattern in the
+// mask.
+#[test]
+fn grow_within_unused_bits_with_overlap() {
+    // To have spare bits, we use a mask size smaller than its block size of 64.
+    let mut mask = InitMask::new(Size::from_bytes(16), true);
+    for i in 0..16 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+
+    // Grow without requiring an additional block, but leave no gap after the current len. Note:
+    // since this is partially out-of-bounds of the current mask, this is case #2 described in the
+    // `set_range` method.
+    let range = (8..24).into();
+    mask.set_range(range, false);
+
+    // We want to make sure the unused bits in the first block are correct
+    for i in 8..24 {
+        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
+    }
+
+    assert_eq!(1, mask.range_as_init_chunks((0..8).into()).count());
+    assert_eq!(2, mask.range_as_init_chunks((0..24).into()).count());
+    assert_eq!(materialized_block_count(&mask), 1);
+}
+
+// Force materialization before a full overwrite: the mask can now become lazy.
+#[test]
+fn grow_mixed_state_within_unused_bits_and_full_overwrite() {
+    // To have spare bits, we use a mask size smaller than its block size of 64.
+    let mut mask = InitMask::new(Size::from_bytes(16), true);
+    let range = (0..16).into();
+    assert!(mask.is_range_initialized(range).is_ok());
+
+    // Force materialization.
+    let range = (8..24).into();
+    mask.set_range(range, false);
+    assert!(mask.is_range_initialized(range).is_err());
+    assert_eq!(materialized_block_count(&mask), 1);
+
+    // Full overwrite, lazy blocks would be enough from now on.
+    let range = (0..32).into();
+    mask.set_range(range, true);
+    assert!(mask.is_range_initialized(range).is_ok());
+
+    assert_eq!(1, mask.range_as_init_chunks((0..32).into()).count());
+    assert_eq!(materialized_block_count(&mask), 0);
+}
+
+// Check that growth outside the current capacity can still be lazy: if the init state doesn't
+// change, we don't need materialized blocks.
+#[test]
+fn grow_same_state_outside_capacity() {
+    // To have spare bits, we use a mask size smaller than its block size of 64.
+    let mut mask = InitMask::new(Size::from_bytes(16), true);
+    for i in 0..16 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+    assert_eq!(materialized_block_count(&mask), 0);
+
+    // Grow to 10 blocks with the same init state.
+    let range = (24..640).into();
+    mask.set_range(range, true);
+
+    assert_eq!(1, mask.range_as_init_chunks((0..640).into()).count());
+    assert_eq!(materialized_block_count(&mask), 0);
+}
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs b/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
index ddd3f3943..318f93e12 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
@@ -14,7 +14,7 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 #[derive(HashStable)]
 pub struct ProvenanceMap<Prov = AllocId> {
     /// Provenance in this map applies from the given offset for an entire pointer-size worth of
-    /// bytes. Two entires in this map are always at least a pointer size apart.
+    /// bytes. Two entries in this map are always at least a pointer size apart.
     ptrs: SortedMap<Size, Prov>,
     /// Provenance in this map only applies to the given single byte.
     /// This map is disjoint from the previous. It will always be empty when
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/tests.rs b/compiler/rustc_middle/src/mir/interpret/allocation/tests.rs
deleted file mode 100644
index c9c3c50c5..000000000
--- a/compiler/rustc_middle/src/mir/interpret/allocation/tests.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-use super::*;
-
-#[test]
-fn uninit_mask() {
-    let mut mask = InitMask::new(Size::from_bytes(500), false);
-    assert!(!mask.get(Size::from_bytes(499)));
-    mask.set_range(alloc_range(Size::from_bytes(499), Size::from_bytes(1)), true);
-    assert!(mask.get(Size::from_bytes(499)));
-    mask.set_range((100..256).into(), true);
-    for i in 0..100 {
-        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
-    }
-    for i in 100..256 {
-        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
-    }
-    for i in 256..499 {
-        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
-    }
-}
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index 1766d7a66..1f8b650e3 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -263,7 +263,8 @@ impl AllocDecodingState {
     }
 
     pub fn new(data_offsets: Vec<u32>) -> Self {
-        let decoding_state = vec![Lock::new(State::Empty); data_offsets.len()];
+        let decoding_state =
+            std::iter::repeat_with(|| Lock::new(State::Empty)).take(data_offsets.len()).collect();
 
         Self { decoding_state, data_offsets }
     }
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index 99cdb769d..2ea8602af 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -10,7 +10,7 @@ use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable};
 use crate::ty::print::{FmtPrinter, Printer};
 use crate::ty::visit::{TypeVisitable, TypeVisitableExt, TypeVisitor};
-use crate::ty::{self, DefIdTree, List, Ty, TyCtxt};
+use crate::ty::{self, List, Ty, TyCtxt};
 use crate::ty::{AdtDef, InstanceDef, ScalarInt, UserTypeAnnotationIndex};
 use crate::ty::{GenericArg, InternalSubsts, SubstsRef};
 
@@ -21,13 +21,13 @@ use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
 use rustc_hir::{self, GeneratorKind, ImplicitSelfKind};
 use rustc_hir::{self as hir, HirId};
 use rustc_session::Session;
-use rustc_target::abi::{Size, VariantIdx};
+use rustc_target::abi::{FieldIdx, Size, VariantIdx};
 
 use polonius_engine::Atom;
 pub use rustc_ast::Mutability;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::graph::dominators::Dominators;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{Idx, IndexSlice, IndexVec};
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
@@ -70,12 +70,19 @@ pub use self::pretty::{
 };
 
 /// Types for locals
-pub type LocalDecls<'tcx> = IndexVec<Local, LocalDecl<'tcx>>;
+pub type LocalDecls<'tcx> = IndexSlice<Local, LocalDecl<'tcx>>;
 
 pub trait HasLocalDecls<'tcx> {
     fn local_decls(&self) -> &LocalDecls<'tcx>;
 }
 
+impl<'tcx> HasLocalDecls<'tcx> for IndexVec<Local, LocalDecl<'tcx>> {
+    #[inline]
+    fn local_decls(&self) -> &LocalDecls<'tcx> {
+        self
+    }
+}
+
 impl<'tcx> HasLocalDecls<'tcx> for LocalDecls<'tcx> {
     #[inline]
     fn local_decls(&self) -> &LocalDecls<'tcx> {
@@ -250,7 +257,7 @@ pub struct Body<'tcx> {
     /// The first local is the return value pointer, followed by `arg_count`
     /// locals for the function arguments, followed by any user-declared
     /// variables and temporaries.
-    pub local_decls: LocalDecls<'tcx>,
+    pub local_decls: IndexVec<Local, LocalDecl<'tcx>>,
 
     /// User type annotations.
     pub user_type_annotations: ty::CanonicalUserTypeAnnotations<'tcx>,
@@ -311,7 +318,7 @@ impl<'tcx> Body<'tcx> {
         source: MirSource<'tcx>,
         basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
         source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>,
-        local_decls: LocalDecls<'tcx>,
+        local_decls: IndexVec<Local, LocalDecl<'tcx>>,
         user_type_annotations: ty::CanonicalUserTypeAnnotations<'tcx>,
         arg_count: usize,
         var_debug_info: Vec<VarDebugInfo<'tcx>>,
@@ -401,8 +408,6 @@ impl<'tcx> Body<'tcx> {
             LocalKind::ReturnPointer
         } else if index < self.arg_count + 1 {
             LocalKind::Arg
-        } else if self.local_decls[local].is_user_variable() {
-            LocalKind::Var
         } else {
             LocalKind::Temp
         }
@@ -572,6 +577,13 @@ impl<T> ClearCrossCrate<T> {
         }
     }
 
+    pub fn as_mut(&mut self) -> ClearCrossCrate<&mut T> {
+        match self {
+            ClearCrossCrate::Clear => ClearCrossCrate::Clear,
+            ClearCrossCrate::Set(v) => ClearCrossCrate::Set(v),
+        }
+    }
+
     pub fn assert_crate_local(self) -> T {
         match self {
             ClearCrossCrate::Clear => bug!("unwrapping cross-crate data"),
@@ -661,9 +673,7 @@ impl Atom for Local {
 /// Classifies locals into categories. See `Body::local_kind`.
 #[derive(Clone, Copy, PartialEq, Eq, Debug, HashStable)]
 pub enum LocalKind {
-    /// User-declared variable binding.
-    Var,
-    /// Compiler-introduced temporary.
+    /// User-declared variable binding or compiler-introduced temporary.
     Temp,
     /// Function argument.
     Arg,
@@ -760,7 +770,7 @@ pub struct LocalDecl<'tcx> {
     pub mutability: Mutability,
 
     // FIXME(matthewjasper) Don't store in this in `Body`
-    pub local_info: Option<Box<LocalInfo<'tcx>>>,
+    pub local_info: ClearCrossCrate<Box<LocalInfo<'tcx>>>,
 
     /// `true` if this is an internal local.
     ///
@@ -778,13 +788,6 @@ pub struct LocalDecl<'tcx> {
     /// generator.
     pub internal: bool,
 
-    /// If this local is a temporary and `is_block_tail` is `Some`,
-    /// then it is a temporary created for evaluation of some
-    /// subexpression of some block's tail expression (with no
-    /// intervening statement context).
-    // FIXME(matthewjasper) Don't store in this in `Body`
-    pub is_block_tail: Option<BlockTailInfo>,
-
     /// The type of this local.
     pub ty: Ty<'tcx>,
 
@@ -890,7 +893,7 @@ pub enum LocalInfo<'tcx> {
     /// The `BindingForm` is solely used for local diagnostics when generating
     /// warnings/errors when compiling the current crate, and therefore it need
     /// not be visible across crates.
-    User(ClearCrossCrate<BindingForm<'tcx>>),
+    User(BindingForm<'tcx>),
     /// A temporary created that references the static with the given `DefId`.
     StaticRef { def_id: DefId, is_thread_local: bool },
     /// A temporary created that references the const with the given `DefId`
@@ -898,13 +901,23 @@ pub enum LocalInfo<'tcx> {
     /// A temporary created during the creation of an aggregate
     /// (e.g. a temporary for `foo` in `MyStruct { my_field: foo }`)
     AggregateTemp,
+    /// A temporary created for evaluation of some subexpression of some block's tail expression
+    /// (with no intervening statement context).
+    // FIXME(matthewjasper) Don't store in this in `Body`
+    BlockTailTemp(BlockTailInfo),
     /// A temporary created during the pass `Derefer` to avoid it's retagging
     DerefTemp,
     /// A temporary created for borrow checking.
     FakeBorrow,
+    /// A local without anything interesting about it.
+    Boring,
 }
 
 impl<'tcx> LocalDecl<'tcx> {
+    pub fn local_info(&self) -> &LocalInfo<'tcx> {
+        &self.local_info.as_ref().assert_crate_local()
+    }
+
     /// Returns `true` only if local is a binding that can itself be
     /// made mutable via the addition of the `mut` keyword, namely
     /// something like the occurrences of `x` in:
@@ -913,15 +926,15 @@ impl<'tcx> LocalDecl<'tcx> {
     /// - or `match ... { C(x) => ... }`
     pub fn can_be_made_mutable(&self) -> bool {
         matches!(
-            self.local_info,
-            Some(box LocalInfo::User(ClearCrossCrate::Set(
+            self.local_info(),
+            LocalInfo::User(
                 BindingForm::Var(VarBindingForm {
                     binding_mode: ty::BindingMode::BindByValue(_),
                     opt_ty_info: _,
                     opt_match_place: _,
                     pat_span: _,
                 }) | BindingForm::ImplicitSelf(ImplicitSelfKind::Imm),
-            )))
+            )
         )
     }
 
@@ -930,15 +943,15 @@ impl<'tcx> LocalDecl<'tcx> {
     /// mutable bindings, but the inverse does not necessarily hold).
     pub fn is_nonref_binding(&self) -> bool {
         matches!(
-            self.local_info,
-            Some(box LocalInfo::User(ClearCrossCrate::Set(
+            self.local_info(),
+            LocalInfo::User(
                 BindingForm::Var(VarBindingForm {
                     binding_mode: ty::BindingMode::BindByValue(_),
                     opt_ty_info: _,
                     opt_match_place: _,
                     pat_span: _,
                 }) | BindingForm::ImplicitSelf(_),
-            )))
+            )
         )
     }
 
@@ -946,38 +959,35 @@ impl<'tcx> LocalDecl<'tcx> {
     /// parameter declared by the user.
     #[inline]
     pub fn is_user_variable(&self) -> bool {
-        matches!(self.local_info, Some(box LocalInfo::User(_)))
+        matches!(self.local_info(), LocalInfo::User(_))
     }
 
     /// Returns `true` if this is a reference to a variable bound in a `match`
     /// expression that is used to access said variable for the guard of the
     /// match arm.
     pub fn is_ref_for_guard(&self) -> bool {
-        matches!(
-            self.local_info,
-            Some(box LocalInfo::User(ClearCrossCrate::Set(BindingForm::RefForGuard)))
-        )
+        matches!(self.local_info(), LocalInfo::User(BindingForm::RefForGuard))
     }
 
     /// Returns `Some` if this is a reference to a static item that is used to
     /// access that static.
     pub fn is_ref_to_static(&self) -> bool {
-        matches!(self.local_info, Some(box LocalInfo::StaticRef { .. }))
+        matches!(self.local_info(), LocalInfo::StaticRef { .. })
     }
 
     /// Returns `Some` if this is a reference to a thread-local static item that is used to
     /// access that static.
     pub fn is_ref_to_thread_local(&self) -> bool {
-        match self.local_info {
-            Some(box LocalInfo::StaticRef { is_thread_local, .. }) => is_thread_local,
+        match self.local_info() {
+            LocalInfo::StaticRef { is_thread_local, .. } => *is_thread_local,
             _ => false,
         }
     }
 
     /// Returns `true` if this is a DerefTemp
     pub fn is_deref_temp(&self) -> bool {
-        match self.local_info {
-            Some(box LocalInfo::DerefTemp) => return true,
+        match self.local_info() {
+            LocalInfo::DerefTemp => return true,
             _ => (),
         }
         return false;
@@ -1001,9 +1011,8 @@ impl<'tcx> LocalDecl<'tcx> {
     pub fn with_source_info(ty: Ty<'tcx>, source_info: SourceInfo) -> Self {
         LocalDecl {
             mutability: Mutability::Mut,
-            local_info: None,
+            local_info: ClearCrossCrate::Set(Box::new(LocalInfo::Boring)),
             internal: false,
-            is_block_tail: None,
             ty,
             user_ty: None,
             source_info,
@@ -1023,20 +1032,11 @@ impl<'tcx> LocalDecl<'tcx> {
         self.mutability = Mutability::Not;
         self
     }
-
-    /// Converts `self` into same `LocalDecl` except tagged as internal temporary.
-    #[inline]
-    pub fn block_tail(mut self, info: BlockTailInfo) -> Self {
-        assert!(self.is_block_tail.is_none());
-        self.is_block_tail = Some(info);
-        self
-    }
 }
 
 #[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
 pub enum VarDebugInfoContents<'tcx> {
-    /// NOTE(eddyb) There's an unenforced invariant that this `Place` is
-    /// based on a `Local`, not a `Static`, and contains no indexing.
+    /// This `Place` only contains projection which satisfy `can_use_in_debuginfo`.
     Place(Place<'tcx>),
     Const(Constant<'tcx>),
     /// The user variable's data is split across several fragments,
@@ -1046,6 +1046,7 @@ pub enum VarDebugInfoContents<'tcx> {
     /// the underlying debuginfo feature this relies on.
     Composite {
         /// Type of the original user variable.
+        /// This cannot contain a union or an enum.
         ty: Ty<'tcx>,
         /// All the parts of the original user variable, which ended
         /// up in disjoint places, due to optimizations.
@@ -1074,17 +1075,16 @@ pub struct VarDebugInfoFragment<'tcx> {
     /// Where in the composite user variable this fragment is,
     /// represented as a "projection" into the composite variable.
     /// At lower levels, this corresponds to a byte/bit range.
-    // NOTE(eddyb) there's an unenforced invariant that this contains
-    // only `Field`s, and not into `enum` variants or `union`s.
-    // FIXME(eddyb) support this for `enum`s by either using DWARF's
+    ///
+    /// This can only contain `PlaceElem::Field`.
+    // FIXME support this for `enum`s by either using DWARF's
     // more advanced control-flow features (unsupported by LLVM?)
     // to match on the discriminant, or by using custom type debuginfo
     // with non-overlapping variants for the composite variable.
     pub projection: Vec<PlaceElem<'tcx>>,
 
     /// Where the data for this fragment can be found.
-    // NOTE(eddyb) There's an unenforced invariant that this `Place` is
-    // contains no indexing (with a non-constant index).
+    /// This `Place` only contains projection which satisfy `can_use_in_debuginfo`.
     pub contents: Place<'tcx>,
 }
 
@@ -1115,6 +1115,11 @@ pub struct VarDebugInfo<'tcx> {
 
     /// Where the data for this user variable is to be found.
     pub value: VarDebugInfoContents<'tcx>,
+
+    /// When present, indicates what argument number this variable is in the function that it
+    /// originated from (starting from 1). Note, if MIR inlining is enabled, then this is the
+    /// argument number in the original function before it was inlined.
+    pub argument_index: Option<u16>,
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -1274,9 +1279,16 @@ impl<'tcx> BasicBlockData<'tcx> {
 }
 
 impl<O> AssertKind<O> {
+    /// Returns true if this an overflow checking assertion controlled by -C overflow-checks.
+    pub fn is_optional_overflow_check(&self) -> bool {
+        use AssertKind::*;
+        use BinOp::*;
+        matches!(self, OverflowNeg(..) | Overflow(Add | Sub | Mul | Shl | Shr, ..))
+    }
+
     /// Getting a description does not require `O` to be printable, and does not
     /// require allocation.
-    /// The caller is expected to handle `BoundsCheck` separately.
+    /// The caller is expected to handle `BoundsCheck` and `MisalignedPointerDereference` separately.
     pub fn description(&self) -> &'static str {
         use AssertKind::*;
         match self {
@@ -1295,7 +1307,9 @@ impl<O> AssertKind<O> {
             ResumedAfterReturn(GeneratorKind::Async(_)) => "`async fn` resumed after completion",
             ResumedAfterPanic(GeneratorKind::Gen) => "generator resumed after panicking",
             ResumedAfterPanic(GeneratorKind::Async(_)) => "`async fn` resumed after panicking",
-            BoundsCheck { .. } => bug!("Unexpected AssertKind"),
+            BoundsCheck { .. } | MisalignedPointerDereference { .. } => {
+                bug!("Unexpected AssertKind")
+            }
         }
     }
 
@@ -1352,6 +1366,13 @@ impl<O> AssertKind<O> {
             Overflow(BinOp::Shl, _, r) => {
                 write!(f, "\"attempt to shift left by `{{}}`, which would overflow\", {:?}", r)
             }
+            MisalignedPointerDereference { required, found } => {
+                write!(
+                    f,
+                    "\"misaligned pointer dereference: address must be a multiple of {{}} but is {{}}\", {:?}, {:?}",
+                    required, found
+                )
+            }
             _ => write!(f, "\"{}\"", self.description()),
         }
     }
@@ -1396,6 +1417,13 @@ impl<O: fmt::Debug> fmt::Debug for AssertKind<O> {
             Overflow(BinOp::Shl, _, r) => {
                 write!(f, "attempt to shift left by `{:#?}`, which would overflow", r)
             }
+            MisalignedPointerDereference { required, found } => {
+                write!(
+                    f,
+                    "misaligned pointer dereference: address must be a multiple of {:?} but is {:?}",
+                    required, found
+                )
+            }
             _ => write!(f, "{}", self.description()),
         }
     }
@@ -1453,6 +1481,9 @@ impl Debug for Statement<'_> {
                 write!(fmt, "discriminant({:?}) = {:?}", place, variant_index)
             }
             Deinit(ref place) => write!(fmt, "Deinit({:?})", place),
+            PlaceMention(ref place) => {
+                write!(fmt, "PlaceMention({:?})", place)
+            }
             AscribeUserType(box (ref place, ref c_ty), ref variance) => {
                 write!(fmt, "AscribeUserType({:?}, {:?}, {:?})", place, variance, c_ty)
             }
@@ -1508,31 +1539,26 @@ impl<V, T> ProjectionElem<V, T> {
     }
 
     /// Returns `true` if this is a `Field` projection with the given index.
-    pub fn is_field_to(&self, f: Field) -> bool {
+    pub fn is_field_to(&self, f: FieldIdx) -> bool {
         matches!(*self, Self::Field(x, _) if x == f)
     }
+
+    /// Returns `true` if this is accepted inside `VarDebugInfoContents::Place`.
+    pub fn can_use_in_debuginfo(&self) -> bool {
+        match self {
+            Self::Deref | Self::Downcast(_, _) | Self::Field(_, _) => true,
+            Self::ConstantIndex { .. }
+            | Self::Index(_)
+            | Self::OpaqueCast(_)
+            | Self::Subslice { .. } => false,
+        }
+    }
 }
 
 /// Alias for projections as they appear in `UserTypeProjection`, where we
 /// need neither the `V` parameter for `Index` nor the `T` for `Field`.
 pub type ProjectionKind = ProjectionElem<(), ()>;
 
-rustc_index::newtype_index! {
-    /// A [newtype'd][wrapper] index type in the MIR [control-flow graph][CFG]
-    ///
-    /// A field (e.g., `f` in `_1.f`) is one variant of [`ProjectionElem`]. Conceptually,
-    /// rustc can identify that a field projection refers to either two different regions of memory
-    /// or the same one between the base and the 'projection element'.
-    /// Read more about projections in the [rustc-dev-guide][mir-datatypes]
-    ///
-    /// [wrapper]: https://rustc-dev-guide.rust-lang.org/appendix/glossary.html#newtype
-    /// [CFG]: https://rustc-dev-guide.rust-lang.org/appendix/background.html#cfg
-    /// [mir-datatypes]: https://rustc-dev-guide.rust-lang.org/mir/index.html#mir-data-types
-    #[derive(HashStable)]
-    #[debug_format = "field[{}]"]
-    pub struct Field {}
-}
-
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub struct PlaceRef<'tcx> {
     pub local: Local,
@@ -1775,7 +1801,7 @@ impl SourceScope {
     /// from the function that was inlined instead of the function call site.
     pub fn lint_root(
         self,
-        source_scopes: &IndexVec<SourceScope, SourceScopeData<'_>>,
+        source_scopes: &IndexSlice<SourceScope, SourceScopeData<'_>>,
     ) -> Option<HirId> {
         let mut data = &source_scopes[self];
         // FIXME(oli-obk): we should be able to just walk the `inlined_parent_scope`, but it
@@ -1795,7 +1821,7 @@ impl SourceScope {
     #[inline]
     pub fn inlined_instance<'tcx>(
         self,
-        source_scopes: &IndexVec<SourceScope, SourceScopeData<'tcx>>,
+        source_scopes: &IndexSlice<SourceScope, SourceScopeData<'tcx>>,
     ) -> Option<ty::Instance<'tcx>> {
         let scope_data = &source_scopes[self];
         if let Some((inlined_instance, _)) = scope_data.inlined {
@@ -1963,7 +1989,8 @@ impl<'tcx> Rvalue<'tcx> {
                 | CastKind::PtrToPtr
                 | CastKind::Pointer(_)
                 | CastKind::PointerFromExposedAddress
-                | CastKind::DynStar,
+                | CastKind::DynStar
+                | CastKind::Transmute,
                 _,
                 _,
             )
@@ -1979,6 +2006,13 @@ impl<'tcx> Rvalue<'tcx> {
 }
 
 impl BorrowKind {
+    pub fn mutability(&self) -> Mutability {
+        match *self {
+            BorrowKind::Shared | BorrowKind::Shallow | BorrowKind::Unique => Mutability::Not,
+            BorrowKind::Mut { .. } => Mutability::Mut,
+        }
+    }
+
     pub fn allows_two_phase_borrow(&self) -> bool {
         match *self {
             BorrowKind::Shared | BorrowKind::Shallow | BorrowKind::Unique => false,
@@ -1995,13 +2029,6 @@ impl BorrowKind {
     }
 }
 
-impl BinOp {
-    pub fn is_checkable(self) -> bool {
-        use self::BinOp::*;
-        matches!(self, Add | Sub | Mul | Shl | Shr)
-    }
-}
-
 impl<'tcx> Debug for Rvalue<'tcx> {
     fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
         use self::Rvalue::*;
@@ -2528,7 +2555,7 @@ impl<'tcx> ConstantKind<'tcx> {
         let parent_substs = if let Some(parent_hir_id) = tcx.hir().opt_parent_id(hir_id)
             && let Some(parent_did) = parent_hir_id.as_owner()
         {
-            InternalSubsts::identity_for_item(tcx, parent_did.to_def_id())
+            InternalSubsts::identity_for_item(tcx, parent_did)
         } else {
             List::empty()
         };
@@ -2557,7 +2584,7 @@ impl<'tcx> ConstantKind<'tcx> {
                 Self::Unevaluated(
                     UnevaluatedConst {
                         def: def.to_global(),
-                        substs: InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
+                        substs: InternalSubsts::identity_for_item(tcx, def.did),
                         promoted: None,
                     },
                     ty,
@@ -2687,12 +2714,17 @@ impl<'tcx> UserTypeProjections {
         self.map_projections(|pat_ty_proj| pat_ty_proj.deref())
     }
 
-    pub fn leaf(self, field: Field) -> Self {
+    pub fn leaf(self, field: FieldIdx) -> Self {
         self.map_projections(|pat_ty_proj| pat_ty_proj.leaf(field))
     }
 
-    pub fn variant(self, adt_def: AdtDef<'tcx>, variant_index: VariantIdx, field: Field) -> Self {
-        self.map_projections(|pat_ty_proj| pat_ty_proj.variant(adt_def, variant_index, field))
+    pub fn variant(
+        self,
+        adt_def: AdtDef<'tcx>,
+        variant_index: VariantIdx,
+        field_index: FieldIdx,
+    ) -> Self {
+        self.map_projections(|pat_ty_proj| pat_ty_proj.variant(adt_def, variant_index, field_index))
     }
 }
 
@@ -2735,7 +2767,7 @@ impl UserTypeProjection {
         self
     }
 
-    pub(crate) fn leaf(mut self, field: Field) -> Self {
+    pub(crate) fn leaf(mut self, field: FieldIdx) -> Self {
         self.projs.push(ProjectionElem::Field(field, ()));
         self
     }
@@ -2744,13 +2776,13 @@ impl UserTypeProjection {
         mut self,
         adt_def: AdtDef<'_>,
         variant_index: VariantIdx,
-        field: Field,
+        field_index: FieldIdx,
     ) -> Self {
         self.projs.push(ProjectionElem::Downcast(
             Some(adt_def.variant(variant_index).name),
             variant_index,
         ));
-        self.projs.push(ProjectionElem::Field(field, ()));
+        self.projs.push(ProjectionElem::Field(field_index, ()));
         self
     }
 }
@@ -3085,7 +3117,7 @@ mod size_asserts {
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
     static_assert_size!(BasicBlockData<'_>, 144);
-    static_assert_size!(LocalDecl<'_>, 56);
+    static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(Statement<'_>, 32);
     static_assert_size!(StatementKind<'_>, 16);
     static_assert_size!(Terminator<'_>, 112);
diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
index 7a05ee2ff..f592f1515 100644
--- a/compiler/rustc_middle/src/mir/mono.rs
+++ b/compiler/rustc_middle/src/mir/mono.rs
@@ -381,7 +381,9 @@ impl<'tcx> CodegenUnit<'tcx> {
                             | InstanceDef::Virtual(..)
                             | InstanceDef::ClosureOnceShim { .. }
                             | InstanceDef::DropGlue(..)
-                            | InstanceDef::CloneShim(..) => None,
+                            | InstanceDef::CloneShim(..)
+                            | InstanceDef::ThreadLocalShim(..)
+                            | InstanceDef::FnPtrAddrShim(..) => None,
                         }
                     }
                     MonoItem::Static(def_id) => def_id.as_local().map(Idx::index),
diff --git a/compiler/rustc_middle/src/mir/patch.rs b/compiler/rustc_middle/src/mir/patch.rs
index 24fe3b472..f62853c3e 100644
--- a/compiler/rustc_middle/src/mir/patch.rs
+++ b/compiler/rustc_middle/src/mir/patch.rs
@@ -12,6 +12,9 @@ pub struct MirPatch<'tcx> {
     new_statements: Vec<(Location, StatementKind<'tcx>)>,
     new_locals: Vec<LocalDecl<'tcx>>,
     resume_block: Option<BasicBlock>,
+    // Only for unreachable in cleanup path.
+    unreachable_cleanup_block: Option<BasicBlock>,
+    terminate_block: Option<BasicBlock>,
     body_span: Span,
     next_local: usize,
 }
@@ -25,14 +28,31 @@ impl<'tcx> MirPatch<'tcx> {
             new_locals: vec![],
             next_local: body.local_decls.len(),
             resume_block: None,
+            unreachable_cleanup_block: None,
+            terminate_block: None,
             body_span: body.span,
         };
 
-        // Check if we already have a resume block
         for (bb, block) in body.basic_blocks.iter_enumerated() {
+            // Check if we already have a resume block
             if let TerminatorKind::Resume = block.terminator().kind && block.statements.is_empty() {
                 result.resume_block = Some(bb);
-                break;
+                continue;
+            }
+
+            // Check if we already have an unreachable block
+            if let TerminatorKind::Unreachable = block.terminator().kind
+                && block.statements.is_empty()
+                && block.is_cleanup
+            {
+                result.unreachable_cleanup_block = Some(bb);
+                continue;
+            }
+
+            // Check if we already have a terminate block
+            if let TerminatorKind::Terminate = block.terminator().kind && block.statements.is_empty() {
+                result.terminate_block = Some(bb);
+                continue;
             }
         }
 
@@ -56,6 +76,40 @@ impl<'tcx> MirPatch<'tcx> {
         bb
     }
 
+    pub fn unreachable_cleanup_block(&mut self) -> BasicBlock {
+        if let Some(bb) = self.unreachable_cleanup_block {
+            return bb;
+        }
+
+        let bb = self.new_block(BasicBlockData {
+            statements: vec![],
+            terminator: Some(Terminator {
+                source_info: SourceInfo::outermost(self.body_span),
+                kind: TerminatorKind::Unreachable,
+            }),
+            is_cleanup: true,
+        });
+        self.unreachable_cleanup_block = Some(bb);
+        bb
+    }
+
+    pub fn terminate_block(&mut self) -> BasicBlock {
+        if let Some(bb) = self.terminate_block {
+            return bb;
+        }
+
+        let bb = self.new_block(BasicBlockData {
+            statements: vec![],
+            terminator: Some(Terminator {
+                source_info: SourceInfo::outermost(self.body_span),
+                kind: TerminatorKind::Terminate,
+            }),
+            is_cleanup: true,
+        });
+        self.terminate_block = Some(bb);
+        bb
+    }
+
     pub fn is_patched(&self, bb: BasicBlock) -> bool {
         self.patch_map[bb].is_some()
     }
@@ -72,28 +126,28 @@ impl<'tcx> MirPatch<'tcx> {
         &mut self,
         ty: Ty<'tcx>,
         span: Span,
-        local_info: Option<Box<LocalInfo<'tcx>>>,
+        local_info: LocalInfo<'tcx>,
     ) -> Local {
         let index = self.next_local;
         self.next_local += 1;
         let mut new_decl = LocalDecl::new(ty, span).internal();
-        new_decl.local_info = local_info;
+        **new_decl.local_info.as_mut().assert_crate_local() = local_info;
         self.new_locals.push(new_decl);
-        Local::new(index as usize)
+        Local::new(index)
     }
 
     pub fn new_temp(&mut self, ty: Ty<'tcx>, span: Span) -> Local {
         let index = self.next_local;
         self.next_local += 1;
         self.new_locals.push(LocalDecl::new(ty, span));
-        Local::new(index as usize)
+        Local::new(index)
     }
 
     pub fn new_internal(&mut self, ty: Ty<'tcx>, span: Span) -> Local {
         let index = self.next_local;
         self.next_local += 1;
         self.new_locals.push(LocalDecl::new(ty, span).internal());
-        Local::new(index as usize)
+        Local::new(index)
     }
 
     pub fn new_block(&mut self, data: BasicBlockData<'tcx>) -> BasicBlock {
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index d8829e3e7..7e5195359 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -123,6 +123,7 @@ fn dump_matched_mir_node<'tcx, F>(
         // see notes on #41697 above
         let def_path =
             ty::print::with_forced_impl_filename_line!(tcx.def_path_str(body.source.def_id()));
+        // ignore-tidy-odd-backticks the literal below is fine
         write!(file, "// MIR for `{}", def_path)?;
         match body.source.promoted {
             None => write!(file, "`")?,
diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs
index b964c1852..cfdf1dcf5 100644
--- a/compiler/rustc_middle/src/mir/query.rs
+++ b/compiler/rustc_middle/src/mir/query.rs
@@ -2,20 +2,20 @@
 
 use crate::mir::{Body, ConstantKind, Promoted};
 use crate::ty::{self, OpaqueHiddenType, Ty, TyCtxt};
-use rustc_data_structures::fx::FxHashSet;
-use rustc_data_structures::vec_map::VecMap;
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_data_structures::unord::UnordSet;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_index::bit_set::BitMatrix;
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_span::Span;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 use smallvec::SmallVec;
 use std::cell::Cell;
 use std::fmt::{self, Debug};
 
-use super::{Field, SourceInfo};
+use super::SourceInfo;
 
 #[derive(Copy, Clone, PartialEq, TyEncodable, TyDecodable, HashStable, Debug)]
 pub enum UnsafetyViolationKind {
@@ -123,7 +123,7 @@ pub struct UnsafetyCheckResult {
     pub violations: Vec<UnsafetyViolation>,
 
     /// Used `unsafe` blocks in this function. This is used for the "unused_unsafe" lint.
-    pub used_unsafe_blocks: FxHashSet<hir::HirId>,
+    pub used_unsafe_blocks: UnordSet<hir::HirId>,
 
     /// This is `Some` iff the item is not a closure.
     pub unused_unsafes: Option<Vec<(hir::HirId, UnusedUnsafe)>>,
@@ -152,7 +152,7 @@ pub struct GeneratorLayout<'tcx> {
 
     /// Which of the above fields are in each variant. Note that one field may
     /// be stored in multiple variants.
-    pub variant_fields: IndexVec<VariantIdx, IndexVec<Field, GeneratorSavedLocal>>,
+    pub variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, GeneratorSavedLocal>>,
 
     /// The source that led to each variant being created (usually, a yield or
     /// await).
@@ -227,9 +227,9 @@ pub struct BorrowCheckResult<'tcx> {
     /// All the opaque types that are restricted to concrete types
     /// by this function. Unlike the value in `TypeckResults`, this has
     /// unerased regions.
-    pub concrete_opaque_types: VecMap<LocalDefId, OpaqueHiddenType<'tcx>>,
+    pub concrete_opaque_types: FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>,
     pub closure_requirements: Option<ClosureRegionRequirements<'tcx>>,
-    pub used_mut_upvars: SmallVec<[Field; 8]>,
+    pub used_mut_upvars: SmallVec<[FieldIdx; 8]>,
     pub tainted_by_errors: Option<ErrorGuaranteed>,
 }
 
@@ -353,7 +353,7 @@ pub enum ConstraintCategory<'tcx> {
     /// like `Foo { field: my_val }`)
     Usage,
     OpaqueType,
-    ClosureUpvar(Field),
+    ClosureUpvar(FieldIdx),
 
     /// A constraint from a user-written predicate
     /// with the provided span, written on the item
@@ -375,7 +375,7 @@ pub enum ConstraintCategory<'tcx> {
 #[derive(TyEncodable, TyDecodable, HashStable, TypeVisitable, TypeFoldable)]
 pub enum ReturnConstraint {
     Normal,
-    ClosureUpvar(Field),
+    ClosureUpvar(FieldIdx),
 }
 
 /// The subject of a `ClosureOutlivesRequirement` -- that is, the thing
@@ -411,10 +411,8 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
     pub fn bind(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Self {
         let inner = tcx.fold_regions(ty, |r, depth| match r.kind() {
             ty::ReVar(vid) => {
-                let br = ty::BoundRegion {
-                    var: ty::BoundVar::new(vid.index()),
-                    kind: ty::BrAnon(vid.as_u32(), None),
-                };
+                let br =
+                    ty::BoundRegion { var: ty::BoundVar::new(vid.index()), kind: ty::BrAnon(None) };
                 tcx.mk_re_late_bound(depth, br)
             }
             _ => bug!("unexpected region in ClosureOutlivesSubjectTy: {r:?}"),
diff --git a/compiler/rustc_middle/src/mir/spanview.rs b/compiler/rustc_middle/src/mir/spanview.rs
index 28a3b51b7..2165403da 100644
--- a/compiler/rustc_middle/src/mir/spanview.rs
+++ b/compiler/rustc_middle/src/mir/spanview.rs
@@ -247,6 +247,7 @@ pub fn statement_kind_name(statement: &Statement<'_>) -> &'static str {
         StorageLive(..) => "StorageLive",
         StorageDead(..) => "StorageDead",
         Retag(..) => "Retag",
+        PlaceMention(..) => "PlaceMention",
         AscribeUserType(..) => "AscribeUserType",
         Coverage(..) => "Coverage",
         Intrinsic(..) => "Intrinsic",
@@ -261,11 +262,10 @@ pub fn terminator_kind_name(term: &Terminator<'_>) -> &'static str {
         Goto { .. } => "Goto",
         SwitchInt { .. } => "SwitchInt",
         Resume => "Resume",
-        Abort => "Abort",
+        Terminate => "Terminate",
         Return => "Return",
         Unreachable => "Unreachable",
         Drop { .. } => "Drop",
-        DropAndReplace { .. } => "DropAndReplace",
         Call { .. } => "Call",
         Assert { .. } => "Assert",
         Yield { .. } => "Yield",
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index ae09562a8..93800d484 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -3,7 +3,7 @@
 //! This is in a dedicated file so that changes to this file can be reviewed more carefully.
 //! The intention is that this file only contains datatype declarations, no code.
 
-use super::{BasicBlock, Constant, Field, Local, SwitchTargets, UserTypeProjection};
+use super::{BasicBlock, Constant, Local, SwitchTargets, UserTypeProjection};
 
 use crate::mir::coverage::{CodeRegion, CoverageKind};
 use crate::traits::Reveal;
@@ -16,7 +16,8 @@ use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::def_id::DefId;
 use rustc_hir::{self as hir};
 use rustc_hir::{self, GeneratorKind};
-use rustc_target::abi::VariantIdx;
+use rustc_index::vec::IndexVec;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 
 use rustc_ast::Mutability;
 use rustc_span::def_id::LocalDefId;
@@ -78,7 +79,8 @@ pub enum MirPhase {
     ///    MIR, this is UB.
     ///  - Retags: If `-Zmir-emit-retag` is enabled, analysis MIR has "implicit" retags in the same way
     ///    that Rust itself has them. Where exactly these are is generally subject to change, and so we
-    ///    don't document this here. Runtime MIR has all retags explicit.
+    ///    don't document this here. Runtime MIR has most retags explicit (though implicit retags
+    ///    can still occur at `Rvalue::{Ref,AddrOf}`).
     ///  - Generator bodies: In analysis MIR, locals may actually be behind a pointer that user code has
     ///    access to. This occurs in generator bodies. Such locals do not behave like other locals,
     ///    because they eg may be aliased in surprising ways. Runtime MIR has no such special locals -
@@ -133,7 +135,6 @@ pub enum AnalysisPhase {
 pub enum RuntimePhase {
     /// In addition to the semantic changes, beginning with this phase, the following variants are
     /// disallowed:
-    /// * [`TerminatorKind::DropAndReplace`]
     /// * [`TerminatorKind::Yield`]
     /// * [`TerminatorKind::GeneratorDrop`]
     /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
@@ -219,6 +220,11 @@ pub enum BorrowKind {
     /// immutable, but not aliasable. This solves the problem. For
     /// simplicity, we don't give users the way to express this
     /// borrow, it's just used when translating closures.
+    ///
+    // FIXME(#112072): This is wrong. Unique borrows are mutable borrows except
+    // that they do not require their pointee to be marked as a mutable.
+    // They should still be treated as mutable borrows in every other way,
+    // e.g. for variance or overlap checking.
     Unique,
 
     /// Data is mutable and not aliasable.
@@ -326,6 +332,15 @@ pub enum StatementKind<'tcx> {
     /// Only `RetagKind::Default` and `RetagKind::FnEntry` are permitted.
     Retag(RetagKind, Box<Place<'tcx>>),
 
+    /// This statement exists to preserve a trace of a scrutinee matched against a wildcard binding.
+    /// This is especially useful for `let _ = PLACE;` bindings that desugar to a single
+    /// `PlaceMention(PLACE)`.
+    ///
+    /// When executed at runtime this is a nop.
+    ///
+    /// Disallowed after drop elaboration.
+    PlaceMention(Box<Place<'tcx>>),
+
     /// Encodes a user's type ascription. These need to be preserved
     /// intact so that NLL can respect them. For example:
     /// ```ignore (illustrative)
@@ -505,15 +520,15 @@ pub struct CopyNonOverlapping<'tcx> {
 ///
 /// A note on unwinding: Panics may occur during the execution of some terminators. Depending on the
 /// `-C panic` flag, this may either cause the program to abort or the call stack to unwind. Such
-/// terminators have a `cleanup: Option<BasicBlock>` field on them. If stack unwinding occurs, then
-/// once the current function is reached, execution continues at the given basic block, if any. If
-/// `cleanup` is `None` then no cleanup is performed, and the stack continues unwinding. This is
-/// equivalent to the execution of a `Resume` terminator.
+/// terminators have a `unwind: UnwindAction` field on them. If stack unwinding occurs, then
+/// once the current function is reached, an action will be taken based on the `unwind` field.
+/// If the action is `Cleanup`, then the execution continues at the given basic block. If the
+/// action is `Continue` then no cleanup is performed, and the stack continues unwinding.
 ///
-/// The basic block pointed to by a `cleanup` field must have its `cleanup` flag set. `cleanup`
-/// basic blocks have a couple restrictions:
-///  1. All `cleanup` fields in them must be `None`.
-///  2. `Return` terminators are not allowed in them. `Abort` and `Unwind` terminators are.
+/// The basic block pointed to by a `Cleanup` unwind action must have its `cleanup` flag set.
+/// `cleanup` basic blocks have a couple restrictions:
+///  1. All `unwind` fields in them must be `UnwindAction::Terminate` or `UnwindAction::Unreachable`.
+///  2. `Return` terminators are not allowed in them. `Terminate` and `Resume` terminators are.
 ///  3. All other basic blocks (in the current body) that are reachable from `cleanup` basic blocks
 ///     must also be `cleanup`. This is a part of the type system and checked statically, so it is
 ///     still an error to have such an edge in the CFG even if it's known that it won't be taken at
@@ -555,11 +570,11 @@ pub enum TerminatorKind<'tcx> {
     /// deaggregation runs.
     Resume,
 
-    /// Indicates that the landing pad is finished and that the process should abort.
+    /// Indicates that the landing pad is finished and that the process should terminate.
     ///
     /// Used to prevent unwinding for foreign items or with `-C unwind=abort`. Only permitted in
     /// cleanup blocks.
-    Abort,
+    Terminate,
 
     /// Returns from the function.
     ///
@@ -594,44 +609,7 @@ pub enum TerminatorKind<'tcx> {
     /// > The drop glue is executed if, among all statements executed within this `Body`, an assignment to
     /// > the place or one of its "parents" occurred more recently than a move out of it. This does not
     /// > consider indirect assignments.
-    Drop { place: Place<'tcx>, target: BasicBlock, unwind: Option<BasicBlock> },
-
-    /// Drops the place and assigns a new value to it.
-    ///
-    /// This first performs the exact same operation as the pre drop-elaboration `Drop` terminator;
-    /// it then additionally assigns the `value` to the `place` as if by an assignment statement.
-    /// This assignment occurs both in the unwind and the regular code paths. The semantics are best
-    /// explained by the elaboration:
-    ///
-    /// ```ignore (MIR)
-    /// BB0 {
-    ///   DropAndReplace(P <- V, goto BB1, unwind BB2)
-    /// }
-    /// ```
-    ///
-    /// becomes
-    ///
-    /// ```ignore (MIR)
-    /// BB0 {
-    ///   Drop(P, goto BB1, unwind BB2)
-    /// }
-    /// BB1 {
-    ///   // P is now uninitialized
-    ///   P <- V
-    /// }
-    /// BB2 {
-    ///   // P is now uninitialized -- its dtor panicked
-    ///   P <- V
-    /// }
-    /// ```
-    ///
-    /// Disallowed after drop elaboration.
-    DropAndReplace {
-        place: Place<'tcx>,
-        value: Operand<'tcx>,
-        target: BasicBlock,
-        unwind: Option<BasicBlock>,
-    },
+    Drop { place: Place<'tcx>, target: BasicBlock, unwind: UnwindAction },
 
     /// Roughly speaking, evaluates the `func` operand and the arguments, and starts execution of
     /// the referred to function. The operand types must match the argument types of the function.
@@ -655,8 +633,8 @@ pub enum TerminatorKind<'tcx> {
         destination: Place<'tcx>,
         /// Where to go after this call returns. If none, the call necessarily diverges.
         target: Option<BasicBlock>,
-        /// Cleanups to be done if the call unwinds.
-        cleanup: Option<BasicBlock>,
+        /// Action to be taken if the call unwinds.
+        unwind: UnwindAction,
         /// `true` if this is from a call in HIR rather than from an overloaded
         /// operator. True for overloaded function call.
         from_hir_call: bool,
@@ -675,14 +653,13 @@ pub enum TerminatorKind<'tcx> {
     /// When overflow checking is disabled and this is run-time MIR (as opposed to compile-time MIR
     /// that is used for CTFE), the following variants of this terminator behave as `goto target`:
     /// - `OverflowNeg(..)`,
-    /// - `Overflow(op, ..)` if op is a "checkable" operation (add, sub, mul, shl, shr, but NOT
-    /// div or rem).
+    /// - `Overflow(op, ..)` if op is add, sub, mul, shl, shr, but NOT div or rem.
     Assert {
         cond: Operand<'tcx>,
         expected: bool,
         msg: AssertMessage<'tcx>,
         target: BasicBlock,
-        cleanup: Option<BasicBlock>,
+        unwind: UnwindAction,
     },
 
     /// Marks a suspend point.
@@ -748,9 +725,8 @@ pub enum TerminatorKind<'tcx> {
         /// in practice, but in order to avoid fragility we want to always
         /// consider it in borrowck. We don't want to accept programs which
         /// pass borrowck only when `panic=abort` or some assertions are disabled
-        /// due to release vs. debug mode builds. This needs to be an `Option` because
-        /// of the `remove_noop_landing_pads` and `abort_unwinding_calls` passes.
-        unwind: Option<BasicBlock>,
+        /// due to release vs. debug mode builds.
+        unwind: UnwindAction,
     },
 
     /// Block ends with an inline assembly block. This is a terminator since
@@ -773,12 +749,31 @@ pub enum TerminatorKind<'tcx> {
         /// diverging (InlineAsmOptions::NORETURN).
         destination: Option<BasicBlock>,
 
-        /// Cleanup to be done if the inline assembly unwinds. This is present
+        /// Action to be taken if the inline assembly unwinds. This is present
         /// if and only if InlineAsmOptions::MAY_UNWIND is set.
-        cleanup: Option<BasicBlock>,
+        unwind: UnwindAction,
     },
 }
 
+/// Action to be taken when a stack unwind happens.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
+#[derive(TypeFoldable, TypeVisitable)]
+pub enum UnwindAction {
+    /// No action is to be taken. Continue unwinding.
+    ///
+    /// This is similar to `Cleanup(bb)` where `bb` does nothing but `Resume`, but they are not
+    /// equivalent, as presence of `Cleanup(_)` will make a frame non-POF.
+    Continue,
+    /// Triggers undefined behavior if unwind happens.
+    Unreachable,
+    /// Terminates the execution if unwind happens.
+    ///
+    /// Depending on the platform and situation this may cause a non-unwindable panic or abort.
+    Terminate,
+    /// Cleanups to be done.
+    Cleanup(BasicBlock),
+}
+
 /// Information about an assertion failure.
 #[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, TypeFoldable, TypeVisitable)]
 pub enum AssertKind<O> {
@@ -789,6 +784,7 @@ pub enum AssertKind<O> {
     RemainderByZero(O),
     ResumedAfterReturn(GeneratorKind),
     ResumedAfterPanic(GeneratorKind),
+    MisalignedPointerDereference { required: O, found: O },
 }
 
 #[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)]
@@ -917,7 +913,15 @@ pub struct Place<'tcx> {
 #[derive(TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
 pub enum ProjectionElem<V, T> {
     Deref,
-    Field(Field, T),
+
+    /// A field (e.g., `f` in `_1.f`) is one variant of [`ProjectionElem`]. Conceptually,
+    /// rustc can identify that a field projection refers to either two different regions of memory
+    /// or the same one between the base and the 'projection element'.
+    /// Read more about projections in the [rustc-dev-guide][mir-datatypes]
+    ///
+    /// [mir-datatypes]: https://rustc-dev-guide.rust-lang.org/mir/index.html#mir-data-types
+    Field(FieldIdx, T),
+
     /// Index into a slice/array.
     ///
     /// Note that this does not also dereference, and so it does not exactly correspond to slice
@@ -1110,11 +1114,7 @@ pub enum Rvalue<'tcx> {
     /// Same as `BinaryOp`, but yields `(T, bool)` with a `bool` indicating an error condition.
     ///
     /// For addition, subtraction, and multiplication on integers the error condition is set when
-    /// the infinite precision result would be unequal to the actual result.
-    ///
-    /// For shift operations on integers the error condition is set when the value of right-hand
-    /// side is greater than or equal to the number of bits in the type of the left-hand side, or
-    /// when the value of right-hand side is negative.
+    /// the infinite precision result would not be equal to the actual result.
     ///
     /// Other combinations of types and operators are unsupported.
     CheckedBinaryOp(BinOp, Box<(Operand<'tcx>, Operand<'tcx>)>),
@@ -1149,7 +1149,7 @@ pub enum Rvalue<'tcx> {
     ///
     /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Generator`. After
     /// generator lowering, `Generator` aggregate kinds are disallowed too.
-    Aggregate(Box<AggregateKind<'tcx>>, Vec<Operand<'tcx>>),
+    Aggregate(Box<AggregateKind<'tcx>>, IndexVec<FieldIdx, Operand<'tcx>>),
 
     /// Transmutes a `*mut u8` into shallow-initialized `Box<T>`.
     ///
@@ -1189,6 +1189,13 @@ pub enum CastKind {
     IntToFloat,
     PtrToPtr,
     FnPtrToPtr,
+    /// Reinterpret the bits of the input as a different type.
+    ///
+    /// MIR is well-formed if the input and output types have different sizes,
+    /// but running a transmute between differently-sized types is UB.
+    ///
+    /// Allowed only in [`MirPhase::Runtime`]; Earlier it's a [`TerminatorKind::Call`].
+    Transmute,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
@@ -1199,11 +1206,11 @@ pub enum AggregateKind<'tcx> {
     Tuple,
 
     /// The second field is the variant index. It's equal to 0 for struct
-    /// and union expressions. The fourth field is
+    /// and union expressions. The last field is the
     /// active field number and is present only for union expressions
     /// -- e.g., for a union expression `SomeUnion { c: .. }`, the
     /// active field index would identity the field `c`
-    Adt(DefId, VariantIdx, SubstsRef<'tcx>, Option<UserTypeAnnotationIndex>, Option<usize>),
+    Adt(DefId, VariantIdx, SubstsRef<'tcx>, Option<UserTypeAnnotationIndex>, Option<FieldIdx>),
 
     Closure(DefId, SubstsRef<'tcx>),
     Generator(DefId, SubstsRef<'tcx>, hir::Movability),
@@ -1280,7 +1287,7 @@ pub enum BinOp {
 mod size_asserts {
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(AggregateKind<'_>, 40);
+    static_assert_size!(AggregateKind<'_>, 32);
     static_assert_size!(Operand<'_>, 24);
     static_assert_size!(Place<'_>, 16);
     static_assert_size!(PlaceElem<'_>, 24);
diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs
index 0aa2c500f..4f00abf7f 100644
--- a/compiler/rustc_middle/src/mir/tcx.rs
+++ b/compiler/rustc_middle/src/mir/tcx.rs
@@ -6,7 +6,7 @@
 use crate::mir::*;
 use crate::ty::{self, Ty, TyCtxt};
 use rustc_hir as hir;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 
 #[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
 pub struct PlaceTy<'tcx> {
@@ -33,7 +33,7 @@ impl<'tcx> PlaceTy<'tcx> {
     ///
     /// Note that the resulting type has not been normalized.
     #[instrument(level = "debug", skip(tcx), ret)]
-    pub fn field_ty(self, tcx: TyCtxt<'tcx>, f: Field) -> Ty<'tcx> {
+    pub fn field_ty(self, tcx: TyCtxt<'tcx>, f: FieldIdx) -> Ty<'tcx> {
         match self.ty.kind() {
             ty::Adt(adt_def, substs) => {
                 let variant_def = match self.variant_index {
@@ -43,7 +43,7 @@ impl<'tcx> PlaceTy<'tcx> {
                         &adt_def.variant(variant_index)
                     }
                 };
-                let field_def = &variant_def.fields[f.index()];
+                let field_def = &variant_def.fields[f];
                 field_def.ty(tcx, substs)
             }
             ty::Tuple(tys) => tys[f.index()],
@@ -61,14 +61,14 @@ impl<'tcx> PlaceTy<'tcx> {
     /// `place_ty.projection_ty_core(tcx, elem, |...| { ... })`
     /// projects `place_ty` onto `elem`, returning the appropriate
     /// `Ty` or downcast variant corresponding to that projection.
-    /// The `handle_field` callback must map a `Field` to its `Ty`,
+    /// The `handle_field` callback must map a `FieldIdx` to its `Ty`,
     /// (which should be trivial when `T` = `Ty`).
     pub fn projection_ty_core<V, T>(
         self,
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
         elem: &ProjectionElem<V, T>,
-        mut handle_field: impl FnMut(&Self, Field, T) -> Ty<'tcx>,
+        mut handle_field: impl FnMut(&Self, FieldIdx, T) -> Ty<'tcx>,
         mut handle_opaque_cast: impl FnMut(&Self, T) -> Ty<'tcx>,
     ) -> PlaceTy<'tcx>
     where
@@ -98,7 +98,7 @@ impl<'tcx> PlaceTy<'tcx> {
                     ty::Array(inner, _) if !from_end => tcx.mk_array(*inner, (to - from) as u64),
                     ty::Array(inner, size) if from_end => {
                         let size = size.eval_target_usize(tcx, param_env);
-                        let len = size - (from as u64) - (to as u64);
+                        let len = size - from - to;
                         tcx.mk_array(*inner, len)
                     }
                     _ => bug!("cannot subslice non-array type: `{:?}`", self),
@@ -116,7 +116,7 @@ impl<'tcx> PlaceTy<'tcx> {
 }
 
 impl<'tcx> Place<'tcx> {
-    pub fn ty_from<D>(
+    pub fn ty_from<D: ?Sized>(
         local: Local,
         projection: &[PlaceElem<'tcx>],
         local_decls: &D,
@@ -132,7 +132,7 @@ impl<'tcx> Place<'tcx> {
             })
     }
 
-    pub fn ty<D>(&self, local_decls: &D, tcx: TyCtxt<'tcx>) -> PlaceTy<'tcx>
+    pub fn ty<D: ?Sized>(&self, local_decls: &D, tcx: TyCtxt<'tcx>) -> PlaceTy<'tcx>
     where
         D: HasLocalDecls<'tcx>,
     {
@@ -141,7 +141,7 @@ impl<'tcx> Place<'tcx> {
 }
 
 impl<'tcx> PlaceRef<'tcx> {
-    pub fn ty<D>(&self, local_decls: &D, tcx: TyCtxt<'tcx>) -> PlaceTy<'tcx>
+    pub fn ty<D: ?Sized>(&self, local_decls: &D, tcx: TyCtxt<'tcx>) -> PlaceTy<'tcx>
     where
         D: HasLocalDecls<'tcx>,
     {
@@ -155,7 +155,7 @@ pub enum RvalueInitializationState {
 }
 
 impl<'tcx> Rvalue<'tcx> {
-    pub fn ty<D>(&self, local_decls: &D, tcx: TyCtxt<'tcx>) -> Ty<'tcx>
+    pub fn ty<D: ?Sized>(&self, local_decls: &D, tcx: TyCtxt<'tcx>) -> Ty<'tcx>
     where
         D: HasLocalDecls<'tcx>,
     {
@@ -164,17 +164,7 @@ impl<'tcx> Rvalue<'tcx> {
             Rvalue::Repeat(ref operand, count) => {
                 tcx.mk_array_with_const_len(operand.ty(local_decls, tcx), count)
             }
-            Rvalue::ThreadLocalRef(did) => {
-                let static_ty = tcx.type_of(did).subst_identity();
-                if tcx.is_mutable_static(did) {
-                    tcx.mk_mut_ptr(static_ty)
-                } else if tcx.is_foreign_item(did) {
-                    tcx.mk_imm_ptr(static_ty)
-                } else {
-                    // FIXME: These things don't *really* have 'static lifetime.
-                    tcx.mk_imm_ref(tcx.lifetimes.re_static, static_ty)
-                }
-            }
+            Rvalue::ThreadLocalRef(did) => tcx.thread_local_ptr_ty(did),
             Rvalue::Ref(reg, bk, ref place) => {
                 let place_ty = place.ty(local_decls, tcx).ty;
                 tcx.mk_ref(reg, ty::TypeAndMut { ty: place_ty, mutbl: bk.to_mutbl_lossy() })
@@ -227,7 +217,7 @@ impl<'tcx> Rvalue<'tcx> {
 }
 
 impl<'tcx> Operand<'tcx> {
-    pub fn ty<D>(&self, local_decls: &D, tcx: TyCtxt<'tcx>) -> Ty<'tcx>
+    pub fn ty<D: ?Sized>(&self, local_decls: &D, tcx: TyCtxt<'tcx>) -> Ty<'tcx>
     where
         D: HasLocalDecls<'tcx>,
     {
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index 6e905224c..2c6126cdd 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -1,6 +1,6 @@
 use smallvec::SmallVec;
 
-use super::{BasicBlock, InlineAsmOperand, Operand, SourceInfo, TerminatorKind};
+use super::{BasicBlock, InlineAsmOperand, Operand, SourceInfo, TerminatorKind, UnwindAction};
 use rustc_ast::InlineAsmTemplatePiece;
 pub use rustc_ast::Mutability;
 use rustc_macros::HashStable;
@@ -118,11 +118,11 @@ impl<'tcx> Terminator<'tcx> {
         self.kind.successors_mut()
     }
 
-    pub fn unwind(&self) -> Option<&Option<BasicBlock>> {
+    pub fn unwind(&self) -> Option<&UnwindAction> {
         self.kind.unwind()
     }
 
-    pub fn unwind_mut(&mut self) -> Option<&mut Option<BasicBlock>> {
+    pub fn unwind_mut(&mut self) -> Option<&mut UnwindAction> {
         self.kind.unwind_mut()
     }
 }
@@ -135,35 +135,33 @@ impl<'tcx> TerminatorKind<'tcx> {
     pub fn successors(&self) -> Successors<'_> {
         use self::TerminatorKind::*;
         match *self {
-            Resume
-            | Abort
-            | GeneratorDrop
-            | Return
-            | Unreachable
-            | Call { target: None, cleanup: None, .. }
-            | InlineAsm { destination: None, cleanup: None, .. } => {
-                None.into_iter().chain((&[]).into_iter().copied())
+            Call { target: Some(t), unwind: UnwindAction::Cleanup(ref u), .. }
+            | Yield { resume: t, drop: Some(ref u), .. }
+            | Drop { target: t, unwind: UnwindAction::Cleanup(ref u), .. }
+            | Assert { target: t, unwind: UnwindAction::Cleanup(ref u), .. }
+            | FalseUnwind { real_target: t, unwind: UnwindAction::Cleanup(ref u) }
+            | InlineAsm { destination: Some(t), unwind: UnwindAction::Cleanup(ref u), .. } => {
+                Some(t).into_iter().chain(slice::from_ref(u).into_iter().copied())
             }
             Goto { target: t }
-            | Call { target: None, cleanup: Some(t), .. }
-            | Call { target: Some(t), cleanup: None, .. }
+            | Call { target: None, unwind: UnwindAction::Cleanup(t), .. }
+            | Call { target: Some(t), unwind: _, .. }
             | Yield { resume: t, drop: None, .. }
-            | DropAndReplace { target: t, unwind: None, .. }
-            | Drop { target: t, unwind: None, .. }
-            | Assert { target: t, cleanup: None, .. }
-            | FalseUnwind { real_target: t, unwind: None }
-            | InlineAsm { destination: Some(t), cleanup: None, .. }
-            | InlineAsm { destination: None, cleanup: Some(t), .. } => {
+            | Drop { target: t, unwind: _, .. }
+            | Assert { target: t, unwind: _, .. }
+            | FalseUnwind { real_target: t, unwind: _ }
+            | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(t), .. }
+            | InlineAsm { destination: Some(t), unwind: _, .. } => {
                 Some(t).into_iter().chain((&[]).into_iter().copied())
             }
-            Call { target: Some(t), cleanup: Some(ref u), .. }
-            | Yield { resume: t, drop: Some(ref u), .. }
-            | DropAndReplace { target: t, unwind: Some(ref u), .. }
-            | Drop { target: t, unwind: Some(ref u), .. }
-            | Assert { target: t, cleanup: Some(ref u), .. }
-            | FalseUnwind { real_target: t, unwind: Some(ref u) }
-            | InlineAsm { destination: Some(t), cleanup: Some(ref u), .. } => {
-                Some(t).into_iter().chain(slice::from_ref(u).into_iter().copied())
+            Resume
+            | Terminate
+            | GeneratorDrop
+            | Return
+            | Unreachable
+            | Call { target: None, unwind: _, .. }
+            | InlineAsm { destination: None, unwind: _, .. } => {
+                None.into_iter().chain((&[]).into_iter().copied())
             }
             SwitchInt { ref targets, .. } => {
                 None.into_iter().chain(targets.targets.iter().copied())
@@ -177,34 +175,34 @@ impl<'tcx> TerminatorKind<'tcx> {
     pub fn successors_mut(&mut self) -> SuccessorsMut<'_> {
         use self::TerminatorKind::*;
         match *self {
-            Resume
-            | Abort
-            | GeneratorDrop
-            | Return
-            | Unreachable
-            | Call { target: None, cleanup: None, .. }
-            | InlineAsm { destination: None, cleanup: None, .. } => None.into_iter().chain(&mut []),
+            Call { target: Some(ref mut t), unwind: UnwindAction::Cleanup(ref mut u), .. }
+            | Yield { resume: ref mut t, drop: Some(ref mut u), .. }
+            | Drop { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. }
+            | Assert { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. }
+            | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u) }
+            | InlineAsm {
+                destination: Some(ref mut t),
+                unwind: UnwindAction::Cleanup(ref mut u),
+                ..
+            } => Some(t).into_iter().chain(slice::from_mut(u)),
             Goto { target: ref mut t }
-            | Call { target: None, cleanup: Some(ref mut t), .. }
-            | Call { target: Some(ref mut t), cleanup: None, .. }
+            | Call { target: None, unwind: UnwindAction::Cleanup(ref mut t), .. }
+            | Call { target: Some(ref mut t), unwind: _, .. }
             | Yield { resume: ref mut t, drop: None, .. }
-            | DropAndReplace { target: ref mut t, unwind: None, .. }
-            | Drop { target: ref mut t, unwind: None, .. }
-            | Assert { target: ref mut t, cleanup: None, .. }
-            | FalseUnwind { real_target: ref mut t, unwind: None }
-            | InlineAsm { destination: Some(ref mut t), cleanup: None, .. }
-            | InlineAsm { destination: None, cleanup: Some(ref mut t), .. } => {
+            | Drop { target: ref mut t, unwind: _, .. }
+            | Assert { target: ref mut t, unwind: _, .. }
+            | FalseUnwind { real_target: ref mut t, unwind: _ }
+            | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(ref mut t), .. }
+            | InlineAsm { destination: Some(ref mut t), unwind: _, .. } => {
                 Some(t).into_iter().chain(&mut [])
             }
-            Call { target: Some(ref mut t), cleanup: Some(ref mut u), .. }
-            | Yield { resume: ref mut t, drop: Some(ref mut u), .. }
-            | DropAndReplace { target: ref mut t, unwind: Some(ref mut u), .. }
-            | Drop { target: ref mut t, unwind: Some(ref mut u), .. }
-            | Assert { target: ref mut t, cleanup: Some(ref mut u), .. }
-            | FalseUnwind { real_target: ref mut t, unwind: Some(ref mut u) }
-            | InlineAsm { destination: Some(ref mut t), cleanup: Some(ref mut u), .. } => {
-                Some(t).into_iter().chain(slice::from_mut(u))
-            }
+            Resume
+            | Terminate
+            | GeneratorDrop
+            | Return
+            | Unreachable
+            | Call { target: None, unwind: _, .. }
+            | InlineAsm { destination: None, unwind: _, .. } => None.into_iter().chain(&mut []),
             SwitchInt { ref mut targets, .. } => None.into_iter().chain(&mut targets.targets),
             FalseEdge { ref mut real_target, ref mut imaginary_target } => {
                 Some(real_target).into_iter().chain(slice::from_mut(imaginary_target))
@@ -212,43 +210,41 @@ impl<'tcx> TerminatorKind<'tcx> {
         }
     }
 
-    pub fn unwind(&self) -> Option<&Option<BasicBlock>> {
+    pub fn unwind(&self) -> Option<&UnwindAction> {
         match *self {
             TerminatorKind::Goto { .. }
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::Yield { .. }
             | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::FalseEdge { .. } => None,
-            TerminatorKind::Call { cleanup: ref unwind, .. }
-            | TerminatorKind::Assert { cleanup: ref unwind, .. }
-            | TerminatorKind::DropAndReplace { ref unwind, .. }
+            TerminatorKind::Call { ref unwind, .. }
+            | TerminatorKind::Assert { ref unwind, .. }
             | TerminatorKind::Drop { ref unwind, .. }
             | TerminatorKind::FalseUnwind { ref unwind, .. }
-            | TerminatorKind::InlineAsm { cleanup: ref unwind, .. } => Some(unwind),
+            | TerminatorKind::InlineAsm { ref unwind, .. } => Some(unwind),
         }
     }
 
-    pub fn unwind_mut(&mut self) -> Option<&mut Option<BasicBlock>> {
+    pub fn unwind_mut(&mut self) -> Option<&mut UnwindAction> {
         match *self {
             TerminatorKind::Goto { .. }
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::Yield { .. }
             | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::FalseEdge { .. } => None,
-            TerminatorKind::Call { cleanup: ref mut unwind, .. }
-            | TerminatorKind::Assert { cleanup: ref mut unwind, .. }
-            | TerminatorKind::DropAndReplace { ref mut unwind, .. }
+            TerminatorKind::Call { ref mut unwind, .. }
+            | TerminatorKind::Assert { ref mut unwind, .. }
             | TerminatorKind::Drop { ref mut unwind, .. }
             | TerminatorKind::FalseUnwind { ref mut unwind, .. }
-            | TerminatorKind::InlineAsm { cleanup: ref mut unwind, .. } => Some(unwind),
+            | TerminatorKind::InlineAsm { ref mut unwind, .. } => Some(unwind),
         }
     }
 
@@ -274,11 +270,17 @@ impl<'tcx> Debug for TerminatorKind<'tcx> {
         let labels = self.fmt_successor_labels();
         assert_eq!(successor_count, labels.len());
 
-        match successor_count {
-            0 => Ok(()),
-
-            1 => write!(fmt, " -> {:?}", self.successors().next().unwrap()),
-
+        let unwind = match self.unwind() {
+            // Not needed or included in successors
+            None | Some(UnwindAction::Continue) | Some(UnwindAction::Cleanup(_)) => None,
+            Some(UnwindAction::Unreachable) => Some("unwind unreachable"),
+            Some(UnwindAction::Terminate) => Some("unwind terminate"),
+        };
+
+        match (successor_count, unwind) {
+            (0, None) => Ok(()),
+            (0, Some(unwind)) => write!(fmt, " -> {}", unwind),
+            (1, None) => write!(fmt, " -> {:?}", self.successors().next().unwrap()),
             _ => {
                 write!(fmt, " -> [")?;
                 for (i, target) in self.successors().enumerate() {
@@ -287,6 +289,9 @@ impl<'tcx> Debug for TerminatorKind<'tcx> {
                     }
                     write!(fmt, "{}: {:?}", labels[i], target)?;
                 }
+                if let Some(unwind) = unwind {
+                    write!(fmt, ", {unwind}")?;
+                }
                 write!(fmt, "]")
             }
         }
@@ -305,13 +310,10 @@ impl<'tcx> TerminatorKind<'tcx> {
             Return => write!(fmt, "return"),
             GeneratorDrop => write!(fmt, "generator_drop"),
             Resume => write!(fmt, "resume"),
-            Abort => write!(fmt, "abort"),
+            Terminate => write!(fmt, "abort"),
             Yield { value, resume_arg, .. } => write!(fmt, "{:?} = yield({:?})", resume_arg, value),
             Unreachable => write!(fmt, "unreachable"),
             Drop { place, .. } => write!(fmt, "drop({:?})", place),
-            DropAndReplace { place, value, .. } => {
-                write!(fmt, "replace({:?} <- {:?})", place, value)
-            }
             Call { func, args, destination, .. } => {
                 write!(fmt, "{:?} = ", destination)?;
                 write!(fmt, "{:?}(", func)?;
@@ -387,7 +389,7 @@ impl<'tcx> TerminatorKind<'tcx> {
     pub fn fmt_successor_labels(&self) -> Vec<Cow<'static, str>> {
         use self::TerminatorKind::*;
         match *self {
-            Return | Resume | Abort | Unreachable | GeneratorDrop => vec![],
+            Return | Resume | Terminate | Unreachable | GeneratorDrop => vec![],
             Goto { .. } => vec!["".into()],
             SwitchInt { ref targets, .. } => targets
                 .values
@@ -395,31 +397,35 @@ impl<'tcx> TerminatorKind<'tcx> {
                 .map(|&u| Cow::Owned(u.to_string()))
                 .chain(iter::once("otherwise".into()))
                 .collect(),
-            Call { target: Some(_), cleanup: Some(_), .. } => {
+            Call { target: Some(_), unwind: UnwindAction::Cleanup(_), .. } => {
                 vec!["return".into(), "unwind".into()]
             }
-            Call { target: Some(_), cleanup: None, .. } => vec!["return".into()],
-            Call { target: None, cleanup: Some(_), .. } => vec!["unwind".into()],
-            Call { target: None, cleanup: None, .. } => vec![],
+            Call { target: Some(_), unwind: _, .. } => vec!["return".into()],
+            Call { target: None, unwind: UnwindAction::Cleanup(_), .. } => vec!["unwind".into()],
+            Call { target: None, unwind: _, .. } => vec![],
             Yield { drop: Some(_), .. } => vec!["resume".into(), "drop".into()],
             Yield { drop: None, .. } => vec!["resume".into()],
-            DropAndReplace { unwind: None, .. } | Drop { unwind: None, .. } => {
-                vec!["return".into()]
-            }
-            DropAndReplace { unwind: Some(_), .. } | Drop { unwind: Some(_), .. } => {
-                vec!["return".into(), "unwind".into()]
+            Drop { unwind: UnwindAction::Cleanup(_), .. } => vec!["return".into(), "unwind".into()],
+            Drop { unwind: _, .. } => vec!["return".into()],
+            Assert { unwind: UnwindAction::Cleanup(_), .. } => {
+                vec!["success".into(), "unwind".into()]
             }
-            Assert { cleanup: None, .. } => vec!["".into()],
-            Assert { .. } => vec!["success".into(), "unwind".into()],
+            Assert { unwind: _, .. } => vec!["success".into()],
             FalseEdge { .. } => vec!["real".into(), "imaginary".into()],
-            FalseUnwind { unwind: Some(_), .. } => vec!["real".into(), "cleanup".into()],
-            FalseUnwind { unwind: None, .. } => vec!["real".into()],
-            InlineAsm { destination: Some(_), cleanup: Some(_), .. } => {
+            FalseUnwind { unwind: UnwindAction::Cleanup(_), .. } => {
+                vec!["real".into(), "unwind".into()]
+            }
+            FalseUnwind { unwind: _, .. } => vec!["real".into()],
+            InlineAsm { destination: Some(_), unwind: UnwindAction::Cleanup(_), .. } => {
                 vec!["return".into(), "unwind".into()]
             }
-            InlineAsm { destination: Some(_), cleanup: None, .. } => vec!["return".into()],
-            InlineAsm { destination: None, cleanup: Some(_), .. } => vec!["unwind".into()],
-            InlineAsm { destination: None, cleanup: None, .. } => vec![],
+            InlineAsm { destination: Some(_), unwind: _, .. } => {
+                vec!["return".into()]
+            }
+            InlineAsm { destination: None, unwind: UnwindAction::Cleanup(_), .. } => {
+                vec!["unwind".into()]
+            }
+            InlineAsm { destination: None, unwind: _, .. } => vec![],
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs
index f37222cb2..7d247eeb6 100644
--- a/compiler/rustc_middle/src/mir/traversal.rs
+++ b/compiler/rustc_middle/src/mir/traversal.rs
@@ -101,7 +101,7 @@ impl<'a, 'tcx> Iterator for Preorder<'a, 'tcx> {
 ///
 /// A Postorder traversal of this graph is `D B C A` or `D C B A`
 pub struct Postorder<'a, 'tcx> {
-    basic_blocks: &'a IndexVec<BasicBlock, BasicBlockData<'tcx>>,
+    basic_blocks: &'a IndexSlice<BasicBlock, BasicBlockData<'tcx>>,
     visited: BitSet<BasicBlock>,
     visit_stack: Vec<(BasicBlock, Successors<'a>)>,
     root_is_start_block: bool,
@@ -109,7 +109,7 @@ pub struct Postorder<'a, 'tcx> {
 
 impl<'a, 'tcx> Postorder<'a, 'tcx> {
     pub fn new(
-        basic_blocks: &'a IndexVec<BasicBlock, BasicBlockData<'tcx>>,
+        basic_blocks: &'a IndexSlice<BasicBlock, BasicBlockData<'tcx>>,
         root: BasicBlock,
     ) -> Postorder<'a, 'tcx> {
         let mut po = Postorder {
@@ -178,17 +178,7 @@ impl<'a, 'tcx> Postorder<'a, 'tcx> {
         // When we yield `B` and call `traverse_successor`, we push `C` to the stack, but
         // since we've already visited `E`, that child isn't added to the stack. The last
         // two iterations yield `C` and finally `A` for a final traversal of [E, D, B, C, A]
-        loop {
-            let bb = if let Some(&mut (_, ref mut iter)) = self.visit_stack.last_mut() {
-                if let Some(bb) = iter.next() {
-                    bb
-                } else {
-                    break;
-                }
-            } else {
-                break;
-            };
-
+        while let Some(&mut (_, ref mut iter)) = self.visit_stack.last_mut() && let Some(bb) = iter.next() {
             if self.visited.insert(bb) {
                 if let Some(term) = &self.basic_blocks[bb].terminator {
                     self.visit_stack.push((bb, term.successors()));
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index 5c056b299..caa5edc32 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -335,12 +335,14 @@ macro_rules! make_mir_visitor {
                         ty::InstanceDef::VTableShim(_def_id) |
                         ty::InstanceDef::ReifyShim(_def_id) |
                         ty::InstanceDef::Virtual(_def_id, _) |
+                        ty::InstanceDef::ThreadLocalShim(_def_id) |
                         ty::InstanceDef::ClosureOnceShim { call_once: _def_id, track_caller: _ } |
                         ty::InstanceDef::DropGlue(_def_id, None) => {}
 
                         ty::InstanceDef::FnPtrShim(_def_id, ty) |
                         ty::InstanceDef::DropGlue(_def_id, Some(ty)) |
-                        ty::InstanceDef::CloneShim(_def_id, ty) => {
+                        ty::InstanceDef::CloneShim(_def_id, ty) |
+                        ty::InstanceDef::FnPtrAddrShim(_def_id, ty) => {
                             // FIXME(eddyb) use a better `TyContext` here.
                             self.visit_ty($(& $mutability)? *ty, TyContext::Location(location));
                         }
@@ -405,6 +407,13 @@ macro_rules! make_mir_visitor {
                     StatementKind::Retag(kind, place) => {
                         self.visit_retag($(& $mutability)? *kind, place, location);
                     }
+                    StatementKind::PlaceMention(place) => {
+                        self.visit_place(
+                            place,
+                            PlaceContext::NonUse(NonUseContext::PlaceMention),
+                            location
+                        );
+                    }
                     StatementKind::AscribeUserType(
                         box (place, user_ty),
                         variance
@@ -453,7 +462,7 @@ macro_rules! make_mir_visitor {
                 match kind {
                     TerminatorKind::Goto { .. } |
                     TerminatorKind::Resume |
-                    TerminatorKind::Abort |
+                    TerminatorKind::Terminate |
                     TerminatorKind::GeneratorDrop |
                     TerminatorKind::Unreachable |
                     TerminatorKind::FalseEdge { .. } |
@@ -495,26 +504,12 @@ macro_rules! make_mir_visitor {
                         );
                     }
 
-                    TerminatorKind::DropAndReplace {
-                        place,
-                        value,
-                        target: _,
-                        unwind: _,
-                    } => {
-                        self.visit_place(
-                            place,
-                            PlaceContext::MutatingUse(MutatingUseContext::Drop),
-                            location
-                        );
-                        self.visit_operand(value, location);
-                    }
-
                     TerminatorKind::Call {
                         func,
                         args,
                         destination,
                         target: _,
-                        cleanup: _,
+                        unwind: _,
                         from_hir_call: _,
                         fn_span: _
                     } => {
@@ -534,7 +529,7 @@ macro_rules! make_mir_visitor {
                         expected: _,
                         msg,
                         target: _,
-                        cleanup: _,
+                        unwind: _,
                     } => {
                         self.visit_operand(cond, location);
                         self.visit_assert_message(msg, location);
@@ -560,7 +555,7 @@ macro_rules! make_mir_visitor {
                         options: _,
                         line_spans: _,
                         destination: _,
-                        cleanup: _,
+                        unwind: _,
                     } => {
                         for op in operands {
                             match op {
@@ -615,6 +610,10 @@ macro_rules! make_mir_visitor {
                     ResumedAfterReturn(_) | ResumedAfterPanic(_) => {
                         // Nothing to visit
                     }
+                    MisalignedPointerDereference { required, found } => {
+                        self.visit_operand(required, location);
+                        self.visit_operand(found, location);
+                    }
                 }
             }
 
@@ -641,8 +640,8 @@ macro_rules! make_mir_visitor {
                             BorrowKind::Shallow => PlaceContext::NonMutatingUse(
                                 NonMutatingUseContext::ShallowBorrow
                             ),
-                            BorrowKind::Unique => PlaceContext::NonMutatingUse(
-                                NonMutatingUseContext::UniqueBorrow
+                            BorrowKind::Unique => PlaceContext::MutatingUse(
+                                MutatingUseContext::Borrow
                             ),
                             BorrowKind::Mut { .. } =>
                                 PlaceContext::MutatingUse(MutatingUseContext::Borrow),
@@ -811,7 +810,6 @@ macro_rules! make_mir_visitor {
                     source_info,
                     internal: _,
                     local_info: _,
-                    is_block_tail: _,
                 } = local_decl;
 
                 self.visit_ty($(& $mutability)? *ty, TyContext::LocalDecl {
@@ -834,6 +832,7 @@ macro_rules! make_mir_visitor {
                     name: _,
                     source_info,
                     value,
+                    argument_index: _,
                 } = var_debug_info;
 
                 self.visit_source_info(source_info);
@@ -1248,8 +1247,6 @@ pub enum NonMutatingUseContext {
     SharedBorrow,
     /// Shallow borrow.
     ShallowBorrow,
-    /// Unique borrow.
-    UniqueBorrow,
     /// AddressOf for *const pointer.
     AddressOf,
     /// Used as base for another place, e.g., `x` in `x.y`. Will not mutate the place.
@@ -1302,6 +1299,8 @@ pub enum NonUseContext {
     AscribeUserTy,
     /// The data of a user variable, for debug info.
     VarDebugInfo,
+    /// PlaceMention statement.
+    PlaceMention,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -1323,9 +1322,7 @@ impl PlaceContext {
         matches!(
             self,
             PlaceContext::NonMutatingUse(
-                NonMutatingUseContext::SharedBorrow
-                    | NonMutatingUseContext::ShallowBorrow
-                    | NonMutatingUseContext::UniqueBorrow
+                NonMutatingUseContext::SharedBorrow | NonMutatingUseContext::ShallowBorrow
             ) | PlaceContext::MutatingUse(MutatingUseContext::Borrow)
         )
     }
diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs
new file mode 100644
index 000000000..7d9aea022
--- /dev/null
+++ b/compiler/rustc_middle/src/query/erase.rs
@@ -0,0 +1,336 @@
+use crate::mir;
+use crate::traits;
+use crate::ty::{self, Ty};
+use std::mem::{size_of, transmute_copy, MaybeUninit};
+
+#[derive(Copy, Clone)]
+pub struct Erased<T: Copy> {
+    // We use `MaybeUninit` here so we can store any value
+    // in `data` since we aren't actually storing a `T`.
+    data: MaybeUninit<T>,
+}
+
+pub trait EraseType: Copy {
+    type Result: Copy;
+}
+
+// Allow `type_alias_bounds` since compilation will fail without `EraseType`.
+#[allow(type_alias_bounds)]
+pub type Erase<T: EraseType> = Erased<impl Copy>;
+
+#[inline(always)]
+pub fn erase<T: EraseType>(src: T) -> Erase<T> {
+    // Ensure the sizes match
+    const {
+        if std::mem::size_of::<T>() != std::mem::size_of::<T::Result>() {
+            panic!("size of T must match erased type T::Result")
+        }
+    };
+
+    Erased::<<T as EraseType>::Result> {
+        // SAFETY: Is it safe to transmute to MaybeUninit for types with the same sizes.
+        data: unsafe { transmute_copy(&src) },
+    }
+}
+
+/// Restores an erased value.
+#[inline(always)]
+pub fn restore<T: EraseType>(value: Erase<T>) -> T {
+    let value: Erased<<T as EraseType>::Result> = value;
+    // SAFETY: Due to the use of impl Trait in `Erase` the only way to safely create an instance
+    // of `Erase` is to call `erase`, so we know that `value.data` is a valid instance of `T` of
+    // the right size.
+    unsafe { transmute_copy(&value.data) }
+}
+
+impl<T> EraseType for &'_ T {
+    type Result = [u8; size_of::<*const ()>()];
+}
+
+impl<T> EraseType for &'_ [T] {
+    type Result = [u8; size_of::<*const [()]>()];
+}
+
+impl<T> EraseType for &'_ ty::List<T> {
+    type Result = [u8; size_of::<*const ()>()];
+}
+
+impl<T> EraseType for Result<&'_ T, traits::query::NoSolution> {
+    type Result = [u8; size_of::<Result<&'static (), traits::query::NoSolution>>()];
+}
+
+impl<T> EraseType for Result<&'_ T, rustc_errors::ErrorGuaranteed> {
+    type Result = [u8; size_of::<Result<&'static (), rustc_errors::ErrorGuaranteed>>()];
+}
+
+impl<T> EraseType for Result<&'_ T, traits::CodegenObligationError> {
+    type Result = [u8; size_of::<Result<&'static (), traits::CodegenObligationError>>()];
+}
+
+impl<T> EraseType for Result<&'_ T, ty::layout::FnAbiError<'_>> {
+    type Result = [u8; size_of::<Result<&'static (), ty::layout::FnAbiError<'static>>>()];
+}
+
+impl<T> EraseType for Result<(&'_ T, rustc_middle::thir::ExprId), rustc_errors::ErrorGuaranteed> {
+    type Result = [u8; size_of::<
+        Result<(&'static (), rustc_middle::thir::ExprId), rustc_errors::ErrorGuaranteed>,
+    >()];
+}
+
+impl EraseType for Result<Option<ty::Instance<'_>>, rustc_errors::ErrorGuaranteed> {
+    type Result =
+        [u8; size_of::<Result<Option<ty::Instance<'static>>, rustc_errors::ErrorGuaranteed>>()];
+}
+
+impl EraseType for Result<Option<ty::Const<'_>>, rustc_errors::ErrorGuaranteed> {
+    type Result =
+        [u8; size_of::<Result<Option<ty::Const<'static>>, rustc_errors::ErrorGuaranteed>>()];
+}
+
+impl EraseType for Result<ty::GenericArg<'_>, traits::query::NoSolution> {
+    type Result = [u8; size_of::<Result<ty::GenericArg<'static>, traits::query::NoSolution>>()];
+}
+
+impl EraseType for Result<bool, ty::layout::LayoutError<'_>> {
+    type Result = [u8; size_of::<Result<bool, ty::layout::LayoutError<'static>>>()];
+}
+
+impl EraseType for Result<rustc_target::abi::TyAndLayout<'_, Ty<'_>>, ty::layout::LayoutError<'_>> {
+    type Result = [u8; size_of::<
+        Result<
+            rustc_target::abi::TyAndLayout<'static, Ty<'static>>,
+            ty::layout::LayoutError<'static>,
+        >,
+    >()];
+}
+
+impl EraseType for Result<ty::Const<'_>, mir::interpret::LitToConstError> {
+    type Result = [u8; size_of::<Result<ty::Const<'static>, mir::interpret::LitToConstError>>()];
+}
+
+impl EraseType for Result<mir::ConstantKind<'_>, mir::interpret::LitToConstError> {
+    type Result =
+        [u8; size_of::<Result<mir::ConstantKind<'static>, mir::interpret::LitToConstError>>()];
+}
+
+impl EraseType for Result<mir::interpret::ConstAlloc<'_>, mir::interpret::ErrorHandled> {
+    type Result = [u8; size_of::<
+        Result<mir::interpret::ConstAlloc<'static>, mir::interpret::ErrorHandled>,
+    >()];
+}
+
+impl EraseType for Result<mir::interpret::ConstValue<'_>, mir::interpret::ErrorHandled> {
+    type Result = [u8; size_of::<
+        Result<mir::interpret::ConstValue<'static>, mir::interpret::ErrorHandled>,
+    >()];
+}
+
+impl EraseType for Result<Option<ty::ValTree<'_>>, mir::interpret::ErrorHandled> {
+    type Result =
+        [u8; size_of::<Result<Option<ty::ValTree<'static>>, mir::interpret::ErrorHandled>>()];
+}
+
+impl EraseType for Result<&'_ ty::List<Ty<'_>>, ty::util::AlwaysRequiresDrop> {
+    type Result =
+        [u8; size_of::<Result<&'static ty::List<Ty<'static>>, ty::util::AlwaysRequiresDrop>>()];
+}
+
+impl<T> EraseType for Option<&'_ T> {
+    type Result = [u8; size_of::<Option<&'static ()>>()];
+}
+
+impl<T> EraseType for Option<&'_ [T]> {
+    type Result = [u8; size_of::<Option<&'static [()]>>()];
+}
+
+impl EraseType for Option<rustc_middle::hir::Owner<'_>> {
+    type Result = [u8; size_of::<Option<rustc_middle::hir::Owner<'static>>>()];
+}
+
+impl EraseType for Option<mir::DestructuredConstant<'_>> {
+    type Result = [u8; size_of::<Option<mir::DestructuredConstant<'static>>>()];
+}
+
+impl EraseType for Option<ty::EarlyBinder<ty::TraitRef<'_>>> {
+    type Result = [u8; size_of::<Option<ty::EarlyBinder<ty::TraitRef<'static>>>>()];
+}
+
+impl EraseType for Option<ty::EarlyBinder<Ty<'_>>> {
+    type Result = [u8; size_of::<Option<ty::EarlyBinder<Ty<'static>>>>()];
+}
+
+impl<T> EraseType for rustc_hir::MaybeOwner<&'_ T> {
+    type Result = [u8; size_of::<rustc_hir::MaybeOwner<&'static ()>>()];
+}
+
+impl<T: EraseType> EraseType for ty::EarlyBinder<T> {
+    type Result = T::Result;
+}
+
+impl EraseType for ty::Binder<'_, ty::FnSig<'_>> {
+    type Result = [u8; size_of::<ty::Binder<'static, ty::FnSig<'static>>>()];
+}
+
+impl<T0, T1> EraseType for (&'_ T0, &'_ T1) {
+    type Result = [u8; size_of::<(&'static (), &'static ())>()];
+}
+
+impl<T0, T1> EraseType for (&'_ T0, &'_ [T1]) {
+    type Result = [u8; size_of::<(&'static (), &'static [()])>()];
+}
+
+macro_rules! trivial {
+    ($($ty:ty),+ $(,)?) => {
+        $(
+            impl EraseType for $ty {
+                type Result = [u8; size_of::<$ty>()];
+            }
+        )*
+    }
+}
+
+trivial! {
+    (),
+    bool,
+    Option<(rustc_span::def_id::DefId, rustc_session::config::EntryFnType)>,
+    Option<rustc_ast::expand::allocator::AllocatorKind>,
+    Option<rustc_attr::ConstStability>,
+    Option<rustc_attr::DefaultBodyStability>,
+    Option<rustc_attr::Stability>,
+    Option<rustc_data_structures::svh::Svh>,
+    Option<rustc_hir::def::DefKind>,
+    Option<rustc_hir::GeneratorKind>,
+    Option<rustc_hir::HirId>,
+    Option<rustc_middle::middle::stability::DeprecationEntry>,
+    Option<rustc_middle::ty::Destructor>,
+    Option<rustc_middle::ty::ImplTraitInTraitData>,
+    Option<rustc_span::def_id::CrateNum>,
+    Option<rustc_span::def_id::DefId>,
+    Option<rustc_span::def_id::LocalDefId>,
+    Option<rustc_span::Span>,
+    Option<rustc_target::spec::PanicStrategy>,
+    Option<usize>,
+    Result<(), rustc_errors::ErrorGuaranteed>,
+    Result<(), rustc_middle::traits::query::NoSolution>,
+    Result<rustc_middle::traits::EvaluationResult, rustc_middle::traits::OverflowError>,
+    rustc_ast::expand::allocator::AllocatorKind,
+    rustc_attr::ConstStability,
+    rustc_attr::DefaultBodyStability,
+    rustc_attr::Deprecation,
+    rustc_attr::Stability,
+    rustc_data_structures::svh::Svh,
+    rustc_errors::ErrorGuaranteed,
+    rustc_hir::Constness,
+    rustc_hir::def_id::DefId,
+    rustc_hir::def_id::DefIndex,
+    rustc_hir::def_id::LocalDefId,
+    rustc_hir::def::DefKind,
+    rustc_hir::Defaultness,
+    rustc_hir::definitions::DefKey,
+    rustc_hir::GeneratorKind,
+    rustc_hir::HirId,
+    rustc_hir::IsAsync,
+    rustc_hir::ItemLocalId,
+    rustc_hir::LangItem,
+    rustc_hir::OwnerId,
+    rustc_hir::Upvar,
+    rustc_index::bit_set::FiniteBitSet<u32>,
+    rustc_middle::middle::dependency_format::Linkage,
+    rustc_middle::middle::exported_symbols::SymbolExportInfo,
+    rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
+    rustc_middle::middle::resolve_bound_vars::ResolvedArg,
+    rustc_middle::middle::stability::DeprecationEntry,
+    rustc_middle::mir::ConstQualifs,
+    rustc_middle::mir::interpret::AllocId,
+    rustc_middle::mir::interpret::ErrorHandled,
+    rustc_middle::mir::interpret::LitToConstError,
+    rustc_middle::thir::ExprId,
+    rustc_middle::traits::CodegenObligationError,
+    rustc_middle::traits::EvaluationResult,
+    rustc_middle::traits::OverflowError,
+    rustc_middle::traits::query::NoSolution,
+    rustc_middle::traits::WellFormedLoc,
+    rustc_middle::ty::adjustment::CoerceUnsizedInfo,
+    rustc_middle::ty::AssocItem,
+    rustc_middle::ty::AssocItemContainer,
+    rustc_middle::ty::BoundVariableKind,
+    rustc_middle::ty::DeducedParamAttrs,
+    rustc_middle::ty::Destructor,
+    rustc_middle::ty::fast_reject::SimplifiedType,
+    rustc_middle::ty::ImplPolarity,
+    rustc_middle::ty::Representability,
+    rustc_middle::ty::ReprOptions,
+    rustc_middle::ty::UnusedGenericParams,
+    rustc_middle::ty::util::AlwaysRequiresDrop,
+    rustc_middle::ty::Visibility<rustc_span::def_id::DefId>,
+    rustc_session::config::CrateType,
+    rustc_session::config::EntryFnType,
+    rustc_session::config::OptLevel,
+    rustc_session::config::SymbolManglingVersion,
+    rustc_session::cstore::CrateDepKind,
+    rustc_session::cstore::ExternCrate,
+    rustc_session::cstore::LinkagePreference,
+    rustc_session::Limits,
+    rustc_session::lint::LintExpectationId,
+    rustc_span::def_id::CrateNum,
+    rustc_span::def_id::DefPathHash,
+    rustc_span::ExpnHash,
+    rustc_span::ExpnId,
+    rustc_span::Span,
+    rustc_span::Symbol,
+    rustc_span::symbol::Ident,
+    rustc_target::spec::PanicStrategy,
+    rustc_type_ir::Variance,
+    u32,
+    usize,
+}
+
+macro_rules! tcx_lifetime {
+    ($($($fake_path:ident)::+),+ $(,)?) => {
+        $(
+            impl<'tcx> EraseType for $($fake_path)::+<'tcx> {
+                type Result = [u8; size_of::<$($fake_path)::+<'static>>()];
+            }
+        )*
+    }
+}
+
+tcx_lifetime! {
+    rustc_middle::hir::Owner,
+    rustc_middle::middle::exported_symbols::ExportedSymbol,
+    rustc_middle::mir::ConstantKind,
+    rustc_middle::mir::DestructuredConstant,
+    rustc_middle::mir::interpret::ConstAlloc,
+    rustc_middle::mir::interpret::ConstValue,
+    rustc_middle::mir::interpret::GlobalId,
+    rustc_middle::mir::interpret::LitToConstInput,
+    rustc_middle::traits::ChalkEnvironmentAndGoal,
+    rustc_middle::traits::query::MethodAutoderefStepsResult,
+    rustc_middle::traits::query::type_op::AscribeUserType,
+    rustc_middle::traits::query::type_op::Eq,
+    rustc_middle::traits::query::type_op::ProvePredicate,
+    rustc_middle::traits::query::type_op::Subtype,
+    rustc_middle::ty::AdtDef,
+    rustc_middle::ty::AliasTy,
+    rustc_middle::ty::Clause,
+    rustc_middle::ty::ClosureTypeInfo,
+    rustc_middle::ty::Const,
+    rustc_middle::ty::DestructuredConst,
+    rustc_middle::ty::ExistentialTraitRef,
+    rustc_middle::ty::FnSig,
+    rustc_middle::ty::GenericArg,
+    rustc_middle::ty::GenericPredicates,
+    rustc_middle::ty::inhabitedness::InhabitedPredicate,
+    rustc_middle::ty::Instance,
+    rustc_middle::ty::InstanceDef,
+    rustc_middle::ty::layout::FnAbiError,
+    rustc_middle::ty::layout::LayoutError,
+    rustc_middle::ty::ParamEnv,
+    rustc_middle::ty::Predicate,
+    rustc_middle::ty::SymbolName,
+    rustc_middle::ty::TraitRef,
+    rustc_middle::ty::Ty,
+    rustc_middle::ty::UnevaluatedConst,
+    rustc_middle::ty::ValTree,
+    rustc_middle::ty::VtblEntry,
+}
diff --git a/compiler/rustc_middle/src/query/keys.rs b/compiler/rustc_middle/src/query/keys.rs
index 78ee8a6a8..23b28ac5c 100644
--- a/compiler/rustc_middle/src/query/keys.rs
+++ b/compiler/rustc_middle/src/query/keys.rs
@@ -12,6 +12,11 @@ use rustc_hir::hir_id::{HirId, OwnerId};
 use rustc_query_system::query::{DefaultCacheSelector, SingleCacheSelector, VecCacheSelector};
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
+use rustc_target::abi::FieldIdx;
+
+/// Placeholder for `CrateNum`'s "local" counterpart
+#[derive(Copy, Clone, Debug)]
+pub struct LocalCrate;
 
 /// The `Key` trait controls what types can legally be used as the key
 /// for a query.
@@ -21,15 +26,11 @@ pub trait Key: Sized {
     //
     //      ...But r-a doesn't support them yet and using a default here causes r-a to not infer
     //      return types of queries which is very annoying. Thus, until r-a support associated
-    //      type defaults, plese restrain from using them here <3
+    //      type defaults, please restrain from using them here <3
     //
     //      r-a issue: <https://github.com/rust-lang/rust-analyzer/issues/13693>
     type CacheSelector;
 
-    /// Given an instance of this key, what crate is it referring to?
-    /// This is used to find the provider.
-    fn query_crate_is_local(&self) -> bool;
-
     /// In the event that a cycle occurs, if no explicit span has been
     /// given for a query with key `self`, what span should we use?
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span;
@@ -45,14 +46,17 @@ pub trait Key: Sized {
     }
 }
 
+pub trait AsLocalKey: Key {
+    type LocalKey;
+
+    /// Given an instance of this key, what crate is it referring to?
+    /// This is used to find the provider.
+    fn as_local_key(&self) -> Option<Self::LocalKey>;
+}
+
 impl Key for () {
     type CacheSelector = SingleCacheSelector;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -61,23 +65,22 @@ impl Key for () {
 impl<'tcx> Key for ty::InstanceDef<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         tcx.def_span(self.def_id())
     }
 }
 
-impl<'tcx> Key for ty::Instance<'tcx> {
-    type CacheSelector = DefaultCacheSelector<Self>;
+impl<'tcx> AsLocalKey for ty::InstanceDef<'tcx> {
+    type LocalKey = Self;
 
     #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
+    fn as_local_key(&self) -> Option<Self::LocalKey> {
+        self.def_id().is_local().then(|| *self)
     }
+}
+
+impl<'tcx> Key for ty::Instance<'tcx> {
+    type CacheSelector = DefaultCacheSelector<Self>;
 
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         tcx.def_span(self.def_id())
@@ -87,11 +90,6 @@ impl<'tcx> Key for ty::Instance<'tcx> {
 impl<'tcx> Key for mir::interpret::GlobalId<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.instance.default_span(tcx)
     }
@@ -100,11 +98,6 @@ impl<'tcx> Key for mir::interpret::GlobalId<'tcx> {
 impl<'tcx> Key for (Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -113,11 +106,6 @@ impl<'tcx> Key for (Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>) {
 impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -126,25 +114,27 @@ impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> {
 impl Key for CrateNum {
     type CacheSelector = VecCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        *self == LOCAL_CRATE
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
 }
 
-impl Key for OwnerId {
-    type CacheSelector = VecCacheSelector<Self>;
+impl AsLocalKey for CrateNum {
+    type LocalKey = LocalCrate;
 
     #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
+    fn as_local_key(&self) -> Option<Self::LocalKey> {
+        (*self == LOCAL_CRATE).then_some(LocalCrate)
     }
+}
+
+impl Key for OwnerId {
+    type CacheSelector = VecCacheSelector<Self>;
+
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.to_def_id().default_span(tcx)
     }
+
     fn key_as_def_id(&self) -> Option<DefId> {
         Some(self.to_def_id())
     }
@@ -153,13 +143,10 @@ impl Key for OwnerId {
 impl Key for LocalDefId {
     type CacheSelector = VecCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.to_def_id().default_span(tcx)
     }
+
     fn key_as_def_id(&self) -> Option<DefId> {
         Some(self.to_def_id())
     }
@@ -168,26 +155,28 @@ impl Key for LocalDefId {
 impl Key for DefId {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         tcx.def_span(*self)
     }
+
     #[inline(always)]
     fn key_as_def_id(&self) -> Option<DefId> {
         Some(*self)
     }
 }
 
-impl Key for ty::WithOptConstParam<LocalDefId> {
-    type CacheSelector = DefaultCacheSelector<Self>;
+impl AsLocalKey for DefId {
+    type LocalKey = LocalDefId;
 
     #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
+    fn as_local_key(&self) -> Option<Self::LocalKey> {
+        self.as_local()
     }
+}
+
+impl Key for ty::WithOptConstParam<LocalDefId> {
+    type CacheSelector = DefaultCacheSelector<Self>;
+
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.did.default_span(tcx)
     }
@@ -196,10 +185,6 @@ impl Key for ty::WithOptConstParam<LocalDefId> {
 impl Key for SimplifiedType {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -208,10 +193,6 @@ impl Key for SimplifiedType {
 impl Key for (DefId, DefId) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.0.krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.1.default_span(tcx)
     }
@@ -220,10 +201,6 @@ impl Key for (DefId, DefId) {
 impl<'tcx> Key for (ty::Instance<'tcx>, LocalDefId) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.0.default_span(tcx)
     }
@@ -232,10 +209,6 @@ impl<'tcx> Key for (ty::Instance<'tcx>, LocalDefId) {
 impl Key for (DefId, LocalDefId) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.0.krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.1.default_span(tcx)
     }
@@ -244,10 +217,6 @@ impl Key for (DefId, LocalDefId) {
 impl Key for (LocalDefId, DefId) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.0.default_span(tcx)
     }
@@ -256,38 +225,27 @@ impl Key for (LocalDefId, DefId) {
 impl Key for (LocalDefId, LocalDefId) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.0.default_span(tcx)
     }
 }
 
-impl Key for (DefId, Option<Ident>) {
+impl Key for (DefId, Ident) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.0.krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         tcx.def_span(self.0)
     }
+
     #[inline(always)]
     fn key_as_def_id(&self) -> Option<DefId> {
         Some(self.0)
     }
 }
 
-impl Key for (DefId, LocalDefId, Ident) {
+impl Key for (LocalDefId, LocalDefId, Ident) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.0.krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.1.default_span(tcx)
     }
@@ -296,34 +254,40 @@ impl Key for (DefId, LocalDefId, Ident) {
 impl Key for (CrateNum, DefId) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.0 == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.1.default_span(tcx)
     }
 }
 
-impl Key for (CrateNum, SimplifiedType) {
-    type CacheSelector = DefaultCacheSelector<Self>;
+impl AsLocalKey for (CrateNum, DefId) {
+    type LocalKey = DefId;
 
     #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.0 == LOCAL_CRATE
+    fn as_local_key(&self) -> Option<Self::LocalKey> {
+        (self.0 == LOCAL_CRATE).then(|| self.1)
     }
+}
+
+impl Key for (CrateNum, SimplifiedType) {
+    type CacheSelector = DefaultCacheSelector<Self>;
+
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
 }
 
-impl Key for (DefId, SimplifiedType) {
-    type CacheSelector = DefaultCacheSelector<Self>;
+impl AsLocalKey for (CrateNum, SimplifiedType) {
+    type LocalKey = SimplifiedType;
 
     #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.0.krate == LOCAL_CRATE
+    fn as_local_key(&self) -> Option<Self::LocalKey> {
+        (self.0 == LOCAL_CRATE).then(|| self.1)
     }
+}
+
+impl Key for (DefId, SimplifiedType) {
+    type CacheSelector = DefaultCacheSelector<Self>;
+
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.0.default_span(tcx)
     }
@@ -332,10 +296,6 @@ impl Key for (DefId, SimplifiedType) {
 impl<'tcx> Key for SubstsRef<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -344,10 +304,6 @@ impl<'tcx> Key for SubstsRef<'tcx> {
 impl<'tcx> Key for (DefId, SubstsRef<'tcx>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.0.krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.0.default_span(tcx)
     }
@@ -356,10 +312,6 @@ impl<'tcx> Key for (DefId, SubstsRef<'tcx>) {
 impl<'tcx> Key for (ty::UnevaluatedConst<'tcx>, ty::UnevaluatedConst<'tcx>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        (self.0).def.did.krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         (self.0).def.did.default_span(tcx)
     }
@@ -368,10 +320,6 @@ impl<'tcx> Key for (ty::UnevaluatedConst<'tcx>, ty::UnevaluatedConst<'tcx>) {
 impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.0.default_span(tcx)
     }
@@ -380,22 +328,14 @@ impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) {
 impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.1.def_id().krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         tcx.def_span(self.1.def_id())
     }
 }
 
-impl<'tcx> Key for (ty::Const<'tcx>, mir::Field) {
+impl<'tcx> Key for (ty::Const<'tcx>, FieldIdx) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -404,10 +344,6 @@ impl<'tcx> Key for (ty::Const<'tcx>, mir::Field) {
 impl<'tcx> Key for mir::interpret::ConstAlloc<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -416,10 +352,6 @@ impl<'tcx> Key for mir::interpret::ConstAlloc<'tcx> {
 impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.def_id().krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         tcx.def_span(self.def_id())
     }
@@ -428,10 +360,6 @@ impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
 impl<'tcx> Key for ty::PolyExistentialTraitRef<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.def_id().krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         tcx.def_span(self.def_id())
     }
@@ -440,10 +368,6 @@ impl<'tcx> Key for ty::PolyExistentialTraitRef<'tcx> {
 impl<'tcx> Key for (ty::PolyTraitRef<'tcx>, ty::PolyTraitRef<'tcx>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.0.def_id().krate == LOCAL_CRATE
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         tcx.def_span(self.0.def_id())
     }
@@ -452,10 +376,6 @@ impl<'tcx> Key for (ty::PolyTraitRef<'tcx>, ty::PolyTraitRef<'tcx>) {
 impl<'tcx> Key for GenericArg<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -464,10 +384,6 @@ impl<'tcx> Key for GenericArg<'tcx> {
 impl<'tcx> Key for mir::ConstantKind<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -476,10 +392,6 @@ impl<'tcx> Key for mir::ConstantKind<'tcx> {
 impl<'tcx> Key for ty::Const<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -488,13 +400,10 @@ impl<'tcx> Key for ty::Const<'tcx> {
 impl<'tcx> Key for Ty<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
+
     fn ty_adt_id(&self) -> Option<DefId> {
         match self.kind() {
             ty::Adt(adt, _) => Some(adt.did()),
@@ -506,10 +415,6 @@ impl<'tcx> Key for Ty<'tcx> {
 impl<'tcx> Key for TyAndLayout<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -518,10 +423,6 @@ impl<'tcx> Key for TyAndLayout<'tcx> {
 impl<'tcx> Key for (Ty<'tcx>, Ty<'tcx>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -530,10 +431,6 @@ impl<'tcx> Key for (Ty<'tcx>, Ty<'tcx>) {
 impl<'tcx> Key for &'tcx ty::List<ty::Predicate<'tcx>> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -542,10 +439,6 @@ impl<'tcx> Key for &'tcx ty::List<ty::Predicate<'tcx>> {
 impl<'tcx> Key for ty::ParamEnv<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -554,10 +447,6 @@ impl<'tcx> Key for ty::ParamEnv<'tcx> {
 impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        self.value.query_crate_is_local()
-    }
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.value.default_span(tcx)
     }
@@ -566,10 +455,6 @@ impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> {
 impl Key for Symbol {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -578,10 +463,6 @@ impl Key for Symbol {
 impl Key for Option<Symbol> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
     fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -589,14 +470,9 @@ impl Key for Option<Symbol> {
 
 /// Canonical query goals correspond to abstract trait operations that
 /// are not tied to any crate in particular.
-impl<'tcx, T> Key for Canonical<'tcx, T> {
+impl<'tcx, T: Clone> Key for Canonical<'tcx, T> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -605,11 +481,6 @@ impl<'tcx, T> Key for Canonical<'tcx, T> {
 impl Key for (Symbol, u32, u32) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -618,11 +489,6 @@ impl Key for (Symbol, u32, u32) {
 impl<'tcx> Key for (DefId, Ty<'tcx>, SubstsRef<'tcx>, ty::ParamEnv<'tcx>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -631,11 +497,6 @@ impl<'tcx> Key for (DefId, Ty<'tcx>, SubstsRef<'tcx>, ty::ParamEnv<'tcx>) {
 impl<'tcx> Key for (ty::Predicate<'tcx>, traits::WellFormedLoc) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -644,11 +505,6 @@ impl<'tcx> Key for (ty::Predicate<'tcx>, traits::WellFormedLoc) {
 impl<'tcx> Key for (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -657,11 +513,6 @@ impl<'tcx> Key for (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
 impl<'tcx> Key for (ty::Instance<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         self.0.default_span(tcx)
     }
@@ -670,11 +521,6 @@ impl<'tcx> Key for (ty::Instance<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
 impl<'tcx> Key for (Ty<'tcx>, ty::ValTree<'tcx>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
     }
@@ -683,11 +529,6 @@ impl<'tcx> Key for (Ty<'tcx>, ty::ValTree<'tcx>) {
 impl Key for HirId {
     type CacheSelector = DefaultCacheSelector<Self>;
 
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
-
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
         tcx.hir().span(*self)
     }
@@ -702,10 +543,6 @@ impl<'tcx> Key for (ValidityRequirement, ty::ParamEnvAnd<'tcx, Ty<'tcx>>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
     // Just forward to `Ty<'tcx>`
-    #[inline(always)]
-    fn query_crate_is_local(&self) -> bool {
-        true
-    }
 
     fn default_span(&self, _: TyCtxt<'_>) -> Span {
         DUMMY_SP
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 5133da342..7a5a16035 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -7,8 +7,9 @@
 use crate::ty::{self, print::describe_as_module, TyCtxt};
 use rustc_span::def_id::LOCAL_CRATE;
 
+pub mod erase;
 mod keys;
-pub use keys::Key;
+pub use keys::{AsLocalKey, Key, LocalCrate};
 
 // Each of these queries corresponds to a function pointer field in the
 // `Providers` struct for requesting a value of that type, and a method
@@ -26,6 +27,15 @@ rustc_queries! {
         desc { "triggering a delay span bug" }
     }
 
+    query registered_tools(_: ()) -> &'tcx ty::RegisteredTools {
+        arena_cache
+        desc { "compute registered tools for crate" }
+    }
+
+    query early_lint_checks(_: ()) -> () {
+        desc { "perform lints prior to macro expansion" }
+    }
+
     query resolutions(_: ()) -> &'tcx ty::ResolverGlobalCtxt {
         feedable
         no_hash
@@ -87,7 +97,7 @@ rustc_queries! {
 
     /// Gives access to the HIR ID for the given `LocalDefId` owner `key` if any.
     ///
-    /// Definitions that were generated with no HIR, would be feeded to return `None`.
+    /// Definitions that were generated with no HIR, would be fed to return `None`.
     query opt_local_def_id_to_hir_id(key: LocalDefId) -> Option<hir::HirId>{
         desc { |tcx| "getting HIR ID of `{}`", tcx.def_path_str(key.to_def_id()) }
         feedable
@@ -182,6 +192,7 @@ rustc_queries! {
     {
         desc { "determine whether the opaque is a type-alias impl trait" }
         separate_provide_extern
+        feedable
     }
 
     query unsizing_params_for_adt(key: DefId) -> &'tcx rustc_index::bit_set::BitSet<u32>
@@ -616,20 +627,26 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    query implied_predicates_of(key: DefId) -> ty::GenericPredicates<'tcx> {
+        desc { |tcx| "computing the implied predicates of `{}`", tcx.def_path_str(key) }
+        cache_on_disk_if { key.is_local() }
+        separate_provide_extern
+    }
+
     /// The `Option<Ident>` is the name of an associated type. If it is `None`, then this query
     /// returns the full set of predicates. If `Some<Ident>`, then the query returns only the
     /// subset of super-predicates that reference traits that define the given associated type.
     /// This is used to avoid cycles in resolving types like `T::Item`.
-    query super_predicates_that_define_assoc_type(key: (DefId, Option<rustc_span::symbol::Ident>)) -> ty::GenericPredicates<'tcx> {
-        desc { |tcx| "computing the super traits of `{}`{}",
+    query super_predicates_that_define_assoc_type(key: (DefId, rustc_span::symbol::Ident)) -> ty::GenericPredicates<'tcx> {
+        desc { |tcx| "computing the super traits of `{}` with associated type name `{}`",
             tcx.def_path_str(key.0),
-            if let Some(assoc_name) = key.1 { format!(" with associated type name `{}`", assoc_name) } else { "".to_string() },
+            key.1
         }
     }
 
     /// To avoid cycles within the predicates of a single item we compute
     /// per-type-parameter predicates for resolving `T::AssocTy`.
-    query type_param_predicates(key: (DefId, LocalDefId, rustc_span::symbol::Ident)) -> ty::GenericPredicates<'tcx> {
+    query type_param_predicates(key: (LocalDefId, LocalDefId, rustc_span::symbol::Ident)) -> ty::GenericPredicates<'tcx> {
         desc { |tcx| "computing the bounds for type parameter `{}`", tcx.hir().ty_param_name(key.1) }
     }
 
@@ -764,7 +781,7 @@ rustc_queries! {
     ///
     /// The map returned for `tcx.impl_item_implementor_ids(impl_id)` would be
     ///`{ trait_f: impl_f, trait_g: impl_g }`
-    query impl_item_implementor_ids(impl_id: DefId) -> &'tcx FxHashMap<DefId, DefId> {
+    query impl_item_implementor_ids(impl_id: DefId) -> &'tcx DefIdMap<DefId> {
         arena_cache
         desc { |tcx| "comparing impl items against trait for `{}`", tcx.def_path_str(impl_id) }
     }
@@ -775,7 +792,7 @@ rustc_queries! {
     /// if `fn_def_id` is the def id of a function defined inside an impl that implements a trait, then it
     /// creates and returns the associated items that correspond to each impl trait in return position
     /// of the implemented trait.
-    query associated_items_for_impl_trait_in_trait(fn_def_id: DefId) -> &'tcx [DefId] {
+    query associated_types_for_impl_traits_in_associated_fn(fn_def_id: DefId) -> &'tcx [DefId] {
         desc { |tcx| "creating associated items for impl trait in trait returned by `{}`", tcx.def_path_str(fn_def_id) }
         cache_on_disk_if { fn_def_id.is_local() }
         separate_provide_extern
@@ -783,10 +800,9 @@ rustc_queries! {
 
     /// Given an impl trait in trait `opaque_ty_def_id`, create and return the corresponding
     /// associated item.
-    query associated_item_for_impl_trait_in_trait(opaque_ty_def_id: LocalDefId) -> LocalDefId {
+    query associated_type_for_impl_trait_in_trait(opaque_ty_def_id: LocalDefId) -> LocalDefId {
         desc { |tcx| "creates the associated item corresponding to the opaque type `{}`", tcx.def_path_str(opaque_ty_def_id.to_def_id()) }
         cache_on_disk_if { true }
-        separate_provide_extern
     }
 
     /// Given an `impl_id`, return the trait it implements.
@@ -906,8 +922,8 @@ rustc_queries! {
     /// The second return value maps from ADTs to ignored derived traits (e.g. Debug and Clone) and
     /// their respective impl (i.e., part of the derive macro)
     query live_symbols_and_ignored_derived_traits(_: ()) -> &'tcx (
-        FxHashSet<LocalDefId>,
-        FxHashMap<LocalDefId, Vec<(DefId, DefId)>>
+        LocalDefIdSet,
+        LocalDefIdMap<Vec<(DefId, DefId)>>
     ) {
         arena_cache
         desc { "finding live symbols in crate" }
@@ -1105,9 +1121,9 @@ rustc_queries! {
         desc { "converting literal to mir constant" }
     }
 
-    query check_match(key: DefId) {
-        desc { |tcx| "match-checking `{}`", tcx.def_path_str(key) }
-        cache_on_disk_if { key.is_local() }
+    query check_match(key: LocalDefId) {
+        desc { |tcx| "match-checking `{}`", tcx.def_path_str(key.to_def_id()) }
+        cache_on_disk_if { true }
     }
 
     /// Performs part of the privacy check and computes effective visibilities.
@@ -1120,7 +1136,7 @@ rustc_queries! {
         desc { "checking for private elements in public interfaces" }
     }
 
-    query reachable_set(_: ()) -> &'tcx FxHashSet<LocalDefId> {
+    query reachable_set(_: ()) -> &'tcx LocalDefIdSet {
         arena_cache
         desc { "reachability" }
     }
@@ -1152,14 +1168,6 @@ rustc_queries! {
         feedable
     }
 
-    /// The `opt_rpitit_info` query returns the pair of the def id of the function where the RPIT
-    /// is defined and the opaque def id if any.
-    query opt_rpitit_info(def_id: DefId) -> Option<ty::ImplTraitInTraitData> {
-        desc { |tcx| "opt_rpitit_info `{}`", tcx.def_path_str(def_id) }
-        cache_on_disk_if { def_id.is_local() }
-        feedable
-    }
-
     /// Gets the span for the definition.
     query def_span(def_id: DefId) -> Span {
         desc { |tcx| "looking up span for `{}`", tcx.def_path_str(def_id) }
@@ -1229,7 +1237,7 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query asm_target_features(def_id: DefId) -> &'tcx FxHashSet<Symbol> {
+    query asm_target_features(def_id: DefId) -> &'tcx FxIndexSet<Symbol> {
         desc { |tcx| "computing target features for inline asm of `{}`", tcx.def_path_str(def_id) }
     }
 
@@ -1324,6 +1332,7 @@ rustc_queries! {
     /// might want to use `reveal_all()` method to change modes.
     query param_env(def_id: DefId) -> ty::ParamEnv<'tcx> {
         desc { |tcx| "computing normalized predicates of `{}`", tcx.def_path_str(def_id) }
+        feedable
     }
 
     /// Like `param_env`, but returns the `ParamEnv` in `Reveal::All` mode.
@@ -1507,10 +1516,6 @@ rustc_queries! {
         desc { "getting traits in scope at a block" }
     }
 
-    query module_reexports(def_id: LocalDefId) -> Option<&'tcx [ModChild]> {
-        desc { |tcx| "looking up reexports of module `{}`", tcx.def_path_str(def_id.to_def_id()) }
-    }
-
     query impl_defaultness(def_id: DefId) -> hir::Defaultness {
         desc { |tcx| "looking up whether `{}` is a default impl", tcx.def_path_str(def_id) }
         cache_on_disk_if { def_id.is_local() }
@@ -1845,7 +1850,7 @@ rustc_queries! {
     query maybe_unused_trait_imports(_: ()) -> &'tcx FxIndexSet<LocalDefId> {
         desc { "fetching potentially unused trait imports" }
     }
-    query names_imported_by_glob_use(def_id: LocalDefId) -> &'tcx FxHashSet<Symbol> {
+    query names_imported_by_glob_use(def_id: LocalDefId) -> &'tcx UnordSet<Symbol> {
         desc { |tcx| "finding names imported by glob use for `{}`", tcx.def_path_str(def_id.to_def_id()) }
     }
 
@@ -2114,7 +2119,7 @@ rustc_queries! {
         desc { "raw operations for metadata file access" }
     }
 
-    query crate_for_resolver((): ()) -> &'tcx Steal<rustc_ast::ast::Crate> {
+    query crate_for_resolver((): ()) -> &'tcx Steal<(rustc_ast::Crate, rustc_ast::AttrVec)> {
         feedable
         no_hash
         desc { "the ast before macro expansion and name resolution" }
@@ -2213,7 +2218,7 @@ rustc_queries! {
     }
 
     /// Used in `super_combine_consts` to ICE if the type of the two consts are definitely not going to end up being
-    /// equal to eachother. This might return `Ok` even if the types are unequal, but will never return `Err` if
+    /// equal to eachother. This might return `Ok` even if the types are not equal, but will never return `Err` if
     /// the types might be equal.
     query check_tys_might_be_eq(arg: Canonical<'tcx, (ty::ParamEnv<'tcx>, Ty<'tcx>, Ty<'tcx>)>) -> Result<(), NoSolution> {
         desc { "check whether two const param are definitely not equal to eachother"}
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index 3b11fab8c..7d79a13d3 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -17,14 +17,14 @@ use rustc_index::newtype_index;
 use rustc_index::vec::IndexVec;
 use rustc_middle::middle::region;
 use rustc_middle::mir::interpret::AllocId;
-use rustc_middle::mir::{self, BinOp, BorrowKind, FakeReadCause, Field, Mutability, UnOp};
+use rustc_middle::mir::{self, BinOp, BorrowKind, FakeReadCause, Mutability, UnOp};
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{self, AdtDef, FnSig, Ty, UpvarSubsts};
 use rustc_middle::ty::{CanonicalUserType, CanonicalUserTypeAnnotation};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{sym, Span, Symbol, DUMMY_SP};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 use rustc_target::asm::InlineAsmRegOrRegClass;
 use std::fmt;
 use std::ops::Index;
@@ -227,6 +227,9 @@ pub enum StmtKind<'tcx> {
 
         /// The lint level for this `let` statement.
         lint_level: LintLevel,
+
+        /// Span of the `let <PAT> = <INIT>` part.
+        span: Span,
     },
 }
 
@@ -366,7 +369,7 @@ pub enum ExprKind<'tcx> {
         /// Variant containing the field.
         variant_index: VariantIdx,
         /// This can be a named (`.foo`) or unnamed (`.0`) field.
-        name: Field,
+        name: FieldIdx,
     },
     /// A *non-overloaded* indexing operation.
     Index {
@@ -491,7 +494,7 @@ pub enum ExprKind<'tcx> {
 /// This is used in struct constructors.
 #[derive(Clone, Debug, HashStable)]
 pub struct FieldExpr {
-    pub name: Field,
+    pub name: FieldIdx,
     pub expr: ExprId,
 }
 
@@ -570,7 +573,7 @@ pub enum BindingMode {
 
 #[derive(Clone, Debug, HashStable)]
 pub struct FieldPat<'tcx> {
-    pub field: Field,
+    pub field: FieldIdx,
     pub pattern: Box<Pat<'tcx>>,
 }
 
@@ -594,6 +597,55 @@ impl<'tcx> Pat<'tcx> {
             _ => None,
         }
     }
+
+    /// Call `f` on every "binding" in a pattern, e.g., on `a` in
+    /// `match foo() { Some(a) => (), None => () }`
+    pub fn each_binding(&self, mut f: impl FnMut(Symbol, BindingMode, Ty<'tcx>, Span)) {
+        self.walk_always(|p| {
+            if let PatKind::Binding { name, mode, ty, .. } = p.kind {
+                f(name, mode, ty, p.span);
+            }
+        });
+    }
+
+    /// Walk the pattern in left-to-right order.
+    ///
+    /// If `it(pat)` returns `false`, the children are not visited.
+    pub fn walk(&self, mut it: impl FnMut(&Pat<'tcx>) -> bool) {
+        self.walk_(&mut it)
+    }
+
+    fn walk_(&self, it: &mut impl FnMut(&Pat<'tcx>) -> bool) {
+        if !it(self) {
+            return;
+        }
+
+        use PatKind::*;
+        match &self.kind {
+            Wild | Range(..) | Binding { subpattern: None, .. } | Constant { .. } => {}
+            AscribeUserType { subpattern, .. }
+            | Binding { subpattern: Some(subpattern), .. }
+            | Deref { subpattern } => subpattern.walk_(it),
+            Leaf { subpatterns } | Variant { subpatterns, .. } => {
+                subpatterns.iter().for_each(|field| field.pattern.walk_(it))
+            }
+            Or { pats } => pats.iter().for_each(|p| p.walk_(it)),
+            Array { box ref prefix, ref slice, box ref suffix }
+            | Slice { box ref prefix, ref slice, box ref suffix } => {
+                prefix.iter().chain(slice.iter()).chain(suffix.iter()).for_each(|p| p.walk_(it))
+            }
+        }
+    }
+
+    /// Walk the pattern in left-to-right order.
+    ///
+    /// If you always want to recurse, prefer this method over `walk`.
+    pub fn walk_always(&self, mut it: impl FnMut(&Pat<'tcx>)) {
+        self.walk(|p| {
+            it(p);
+            true
+        })
+    }
 }
 
 impl<'tcx> IntoDiagnosticArg for Pat<'tcx> {
@@ -784,7 +836,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
                             if let PatKind::Wild = p.pattern.kind {
                                 continue;
                             }
-                            let name = variant.fields[p.field.index()].name;
+                            let name = variant.fields[p.field].name;
                             write!(f, "{}{}: {}", start_or_comma(), name, p.pattern)?;
                             printed += 1;
                         }
@@ -879,7 +931,7 @@ mod size_asserts {
     static_assert_size!(ExprKind<'_>, 40);
     static_assert_size!(Pat<'_>, 72);
     static_assert_size!(PatKind<'_>, 56);
-    static_assert_size!(Stmt<'_>, 48);
-    static_assert_size!(StmtKind<'_>, 40);
+    static_assert_size!(Stmt<'_>, 56);
+    static_assert_size!(StmtKind<'_>, 48);
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_middle/src/thir/visit.rs b/compiler/rustc_middle/src/thir/visit.rs
index 79a0e75aa..5614528c4 100644
--- a/compiler/rustc_middle/src/thir/visit.rs
+++ b/compiler/rustc_middle/src/thir/visit.rs
@@ -175,6 +175,7 @@ pub fn walk_stmt<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, stmt: &Stm
             ref pattern,
             lint_level: _,
             else_block,
+            span: _,
         } => {
             if let Some(init) = initializer {
                 visitor.visit_expr(&visitor.thir()[*init]);
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index 6231dd9b6..91f07389f 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -305,8 +305,6 @@ pub enum ObligationCauseCode<'tcx> {
     SizedReturnType,
     /// Yield type must be `Sized`.
     SizedYieldType,
-    /// Box expression result type must be `Sized`.
-    SizedBoxType,
     /// Inline asm operand type must be `Sized`.
     InlineAsmSized,
     /// `[expr; N]` requires `type_of(expr): Copy`.
@@ -699,9 +697,9 @@ impl<'tcx, N> ImplSource<'tcx, N> {
     }
 
     pub fn borrow_nested_obligations(&self) -> &[N] {
-        match &self {
-            ImplSource::UserDefined(i) => &i.nested[..],
-            ImplSource::Param(n, _) => &n,
+        match self {
+            ImplSource::UserDefined(i) => &i.nested,
+            ImplSource::Param(n, _) => n,
             ImplSource::Builtin(i) => &i.nested,
             ImplSource::AutoImpl(d) => &d.nested,
             ImplSource::Closure(c) => &c.nested,
@@ -715,6 +713,23 @@ impl<'tcx, N> ImplSource<'tcx, N> {
         }
     }
 
+    pub fn borrow_nested_obligations_mut(&mut self) -> &mut [N] {
+        match self {
+            ImplSource::UserDefined(i) => &mut i.nested,
+            ImplSource::Param(n, _) => n,
+            ImplSource::Builtin(i) => &mut i.nested,
+            ImplSource::AutoImpl(d) => &mut d.nested,
+            ImplSource::Closure(c) => &mut c.nested,
+            ImplSource::Generator(c) => &mut c.nested,
+            ImplSource::Future(c) => &mut c.nested,
+            ImplSource::Object(d) => &mut d.nested,
+            ImplSource::FnPointer(d) => &mut d.nested,
+            ImplSource::TraitAlias(d) => &mut d.nested,
+            ImplSource::TraitUpcasting(d) => &mut d.nested,
+            ImplSource::ConstDestruct(i) => &mut i.nested,
+        }
+    }
+
     pub fn map<M, F>(self, f: F) -> ImplSource<'tcx, M>
     where
         F: FnMut(N) -> M,
@@ -899,6 +914,9 @@ pub enum ObjectSafetyViolation {
     /// (e.g., `trait Foo : Bar<Self>`).
     SupertraitSelf(SmallVec<[Span; 1]>),
 
+    // Supertrait has a non-lifetime `for<T>` binder.
+    SupertraitNonLifetimeBinder(SmallVec<[Span; 1]>),
+
     /// Method has something illegal.
     Method(Symbol, MethodViolationCode, Span),
 
@@ -921,6 +939,9 @@ impl ObjectSafetyViolation {
                         .into()
                 }
             }
+            ObjectSafetyViolation::SupertraitNonLifetimeBinder(_) => {
+                "where clause cannot reference non-lifetime `for<...>` variables".into()
+            }
             ObjectSafetyViolation::Method(name, MethodViolationCode::StaticMethod(_), _) => {
                 format!("associated function `{}` has no `self` parameter", name).into()
             }
@@ -971,7 +992,9 @@ impl ObjectSafetyViolation {
 
     pub fn solution(&self, err: &mut Diagnostic) {
         match self {
-            ObjectSafetyViolation::SizedSelf(_) | ObjectSafetyViolation::SupertraitSelf(_) => {}
+            ObjectSafetyViolation::SizedSelf(_)
+            | ObjectSafetyViolation::SupertraitSelf(_)
+            | ObjectSafetyViolation::SupertraitNonLifetimeBinder(..) => {}
             ObjectSafetyViolation::Method(
                 name,
                 MethodViolationCode::StaticMethod(Some((add_self_sugg, make_sized_sugg))),
@@ -1025,7 +1048,8 @@ impl ObjectSafetyViolation {
         // diagnostics use a `note` instead of a `span_label`.
         match self {
             ObjectSafetyViolation::SupertraitSelf(spans)
-            | ObjectSafetyViolation::SizedSelf(spans) => spans.clone(),
+            | ObjectSafetyViolation::SizedSelf(spans)
+            | ObjectSafetyViolation::SupertraitNonLifetimeBinder(spans) => spans.clone(),
             ObjectSafetyViolation::AssocConst(_, span)
             | ObjectSafetyViolation::GAT(_, span)
             | ObjectSafetyViolation::Method(_, _, span)
diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs
index bd43867a3..fef2be133 100644
--- a/compiler/rustc_middle/src/traits/solve.rs
+++ b/compiler/rustc_middle/src/traits/solve.rs
@@ -1,12 +1,113 @@
 use std::ops::ControlFlow;
 
 use rustc_data_structures::intern::Interned;
+use rustc_query_system::cache::Cache;
 
-use crate::infer::canonical::QueryRegionConstraints;
+use crate::infer::canonical::{CanonicalVarValues, QueryRegionConstraints};
+use crate::traits::query::NoSolution;
+use crate::traits::Canonical;
 use crate::ty::{
-    FallibleTypeFolder, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeVisitable, TypeVisitor,
+    self, FallibleTypeFolder, ToPredicate, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeVisitable,
+    TypeVisitor,
 };
 
+pub type EvaluationCache<'tcx> = Cache<CanonicalGoal<'tcx>, QueryResult<'tcx>>;
+
+/// A goal is a statement, i.e. `predicate`, we want to prove
+/// given some assumptions, i.e. `param_env`.
+///
+/// Most of the time the `param_env` contains the `where`-bounds of the function
+/// we're currently typechecking while the `predicate` is some trait bound.
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
+pub struct Goal<'tcx, P> {
+    pub predicate: P,
+    pub param_env: ty::ParamEnv<'tcx>,
+}
+
+impl<'tcx, P> Goal<'tcx, P> {
+    pub fn new(
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+        predicate: impl ToPredicate<'tcx, P>,
+    ) -> Goal<'tcx, P> {
+        Goal { param_env, predicate: predicate.to_predicate(tcx) }
+    }
+
+    /// Updates the goal to one with a different `predicate` but the same `param_env`.
+    pub fn with<Q>(self, tcx: TyCtxt<'tcx>, predicate: impl ToPredicate<'tcx, Q>) -> Goal<'tcx, Q> {
+        Goal { param_env: self.param_env, predicate: predicate.to_predicate(tcx) }
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
+pub struct Response<'tcx> {
+    pub certainty: Certainty,
+    pub var_values: CanonicalVarValues<'tcx>,
+    /// Additional constraints returned by this query.
+    pub external_constraints: ExternalConstraints<'tcx>,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
+pub enum Certainty {
+    Yes,
+    Maybe(MaybeCause),
+}
+
+impl Certainty {
+    pub const AMBIGUOUS: Certainty = Certainty::Maybe(MaybeCause::Ambiguity);
+
+    /// Use this function to merge the certainty of multiple nested subgoals.
+    ///
+    /// Given an impl like `impl<T: Foo + Bar> Baz for T {}`, we have 2 nested
+    /// subgoals whenever we use the impl as a candidate: `T: Foo` and `T: Bar`.
+    /// If evaluating `T: Foo` results in ambiguity and `T: Bar` results in
+    /// success, we merge these two responses. This results in ambiguity.
+    ///
+    /// If we unify ambiguity with overflow, we return overflow. This doesn't matter
+    /// inside of the solver as we distinguish ambiguity from overflow. It does
+    /// however matter for diagnostics. If `T: Foo` resulted in overflow and `T: Bar`
+    /// in ambiguity without changing the inference state, we still want to tell the
+    /// user that `T: Baz` results in overflow.
+    pub fn unify_with(self, other: Certainty) -> Certainty {
+        match (self, other) {
+            (Certainty::Yes, Certainty::Yes) => Certainty::Yes,
+            (Certainty::Yes, Certainty::Maybe(_)) => other,
+            (Certainty::Maybe(_), Certainty::Yes) => self,
+            (Certainty::Maybe(MaybeCause::Ambiguity), Certainty::Maybe(MaybeCause::Ambiguity)) => {
+                Certainty::Maybe(MaybeCause::Ambiguity)
+            }
+            (Certainty::Maybe(MaybeCause::Ambiguity), Certainty::Maybe(MaybeCause::Overflow))
+            | (Certainty::Maybe(MaybeCause::Overflow), Certainty::Maybe(MaybeCause::Ambiguity))
+            | (Certainty::Maybe(MaybeCause::Overflow), Certainty::Maybe(MaybeCause::Overflow)) => {
+                Certainty::Maybe(MaybeCause::Overflow)
+            }
+        }
+    }
+}
+
+/// Why we failed to evaluate a goal.
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
+pub enum MaybeCause {
+    /// We failed due to ambiguity. This ambiguity can either
+    /// be a true ambiguity, i.e. there are multiple different answers,
+    /// or we hit a case where we just don't bother, e.g. `?x: Trait` goals.
+    Ambiguity,
+    /// We gave up due to an overflow, most often by hitting the recursion limit.
+    Overflow,
+}
+
+pub type CanonicalGoal<'tcx, T = ty::Predicate<'tcx>> = Canonical<'tcx, Goal<'tcx, T>>;
+
+pub type CanonicalResponse<'tcx> = Canonical<'tcx, Response<'tcx>>;
+
+/// The result of evaluating a canonical query.
+///
+/// FIXME: We use a different type than the existing canonical queries. This is because
+/// we need to add a `Certainty` for `overflow` and may want to restructure this code without
+/// having to worry about changes to currently used code. Once we've made progress on this
+/// solver, merge the two responses again.
+pub type QueryResult<'tcx> = Result<CanonicalResponse<'tcx>, NoSolution>;
+
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Hash)]
 pub struct ExternalConstraints<'tcx>(pub(crate) Interned<'tcx, ExternalConstraintsData<'tcx>>);
 
@@ -14,7 +115,7 @@ impl<'tcx> std::ops::Deref for ExternalConstraints<'tcx> {
     type Target = ExternalConstraintsData<'tcx>;
 
     fn deref(&self) -> &Self::Target {
-        &*self.0
+        &self.0
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/_match.rs b/compiler/rustc_middle/src/ty/_match.rs
index df9aa765d..468c2c818 100644
--- a/compiler/rustc_middle/src/ty/_match.rs
+++ b/compiler/rustc_middle/src/ty/_match.rs
@@ -37,10 +37,6 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
         self.tcx
     }
 
-    fn intercrate(&self) -> bool {
-        false
-    }
-
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
         self.param_env
     }
@@ -48,10 +44,6 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
         true
     } // irrelevant
 
-    fn mark_ambiguous(&mut self) {
-        bug!()
-    }
-
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
         _: ty::Variance,
diff --git a/compiler/rustc_middle/src/ty/adjustment.rs b/compiler/rustc_middle/src/ty/adjustment.rs
index 8ce06404d..cd0f7e8da 100644
--- a/compiler/rustc_middle/src/ty/adjustment.rs
+++ b/compiler/rustc_middle/src/ty/adjustment.rs
@@ -3,6 +3,7 @@ use rustc_hir as hir;
 use rustc_hir::lang_items::LangItem;
 use rustc_macros::HashStable;
 use rustc_span::Span;
+use rustc_target::abi::FieldIdx;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
 pub enum PointerCast {
@@ -208,5 +209,5 @@ pub struct CoerceUnsizedInfo {
 #[derive(Clone, Copy, TyEncodable, TyDecodable, Debug, HashStable)]
 pub enum CustomCoerceUnsized {
     /// Records the index of the field being coerced.
-    Struct(usize),
+    Struct(FieldIdx),
 }
diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs
index ec21030b3..3a03c0901 100644
--- a/compiler/rustc_middle/src/ty/adt.rs
+++ b/compiler/rustc_middle/src/ty/adt.rs
@@ -10,11 +10,11 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, DefKind, Res};
 use rustc_hir::def_id::DefId;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_query_system::ich::StableHashingContext;
 use rustc_session::DataTypeKind;
 use rustc_span::symbol::sym;
-use rustc_target::abi::{ReprOptions, VariantIdx};
+use rustc_target::abi::{ReprOptions, VariantIdx, FIRST_VARIANT};
 
 use std::cell::RefCell;
 use std::cmp::Ordering;
@@ -168,7 +168,7 @@ impl<'tcx> AdtDef<'tcx> {
     }
 
     #[inline]
-    pub fn variants(self) -> &'tcx IndexVec<VariantIdx, VariantDef> {
+    pub fn variants(self) -> &'tcx IndexSlice<VariantIdx, VariantDef> {
         &self.0.0.variants
     }
 
@@ -228,7 +228,7 @@ impl AdtDefData {
             AdtKind::Struct => AdtFlags::IS_STRUCT,
         };
 
-        if kind == AdtKind::Struct && variants[VariantIdx::new(0)].ctor.is_some() {
+        if kind == AdtKind::Struct && variants[FIRST_VARIANT].ctor.is_some() {
             flags |= AdtFlags::HAS_CTOR;
         }
 
@@ -357,7 +357,7 @@ impl<'tcx> AdtDef<'tcx> {
     /// Asserts this is a struct or union and returns its unique variant.
     pub fn non_enum_variant(self) -> &'tcx VariantDef {
         assert!(self.is_struct() || self.is_union());
-        &self.variant(VariantIdx::new(0))
+        &self.variant(FIRST_VARIANT)
     }
 
     #[inline]
@@ -493,7 +493,7 @@ impl<'tcx> AdtDef<'tcx> {
 
     #[inline]
     pub fn variant_range(self) -> Range<VariantIdx> {
-        VariantIdx::new(0)..VariantIdx::new(self.variants().len())
+        FIRST_VARIANT..self.variants().next_index()
     }
 
     /// Computes the discriminant value used by a specific variant.
diff --git a/compiler/rustc_middle/src/ty/assoc.rs b/compiler/rustc_middle/src/ty/assoc.rs
index f1a9e50a4..090b76932 100644
--- a/compiler/rustc_middle/src/ty/assoc.rs
+++ b/compiler/rustc_middle/src/ty/assoc.rs
@@ -1,6 +1,6 @@
 pub use self::AssocItemContainer::*;
 
-use crate::ty::{self, DefIdTree};
+use crate::ty;
 use rustc_data_structures::sorted_map::SortedIndexMultiMap;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Namespace};
@@ -30,6 +30,11 @@ pub struct AssocItem {
     /// Whether this is a method with an explicit self
     /// as its first parameter, allowing method calls.
     pub fn_has_self_parameter: bool,
+
+    /// `Some` if the associated item (an associated type) comes from the
+    /// return-position `impl Trait` in trait desugaring. The `ImplTraitInTraitData`
+    /// provides additional information about its source.
+    pub opt_rpitit_info: Option<ty::ImplTraitInTraitData>,
 }
 
 impl AssocItem {
diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs
index dc2bd54b7..f29bf92b0 100644
--- a/compiler/rustc_middle/src/ty/closure.rs
+++ b/compiler/rustc_middle/src/ty/closure.rs
@@ -158,12 +158,12 @@ impl<'tcx> CapturedPlace<'tcx> {
         for proj in self.place.projections.iter() {
             match proj.kind {
                 HirProjectionKind::Field(idx, variant) => match ty.kind() {
-                    ty::Tuple(_) => write!(&mut symbol, "__{}", idx).unwrap(),
+                    ty::Tuple(_) => write!(&mut symbol, "__{}", idx.index()).unwrap(),
                     ty::Adt(def, ..) => {
                         write!(
                             &mut symbol,
                             "__{}",
-                            def.variant(variant).fields[idx as usize].name.as_str(),
+                            def.variant(variant).fields[idx].name.as_str(),
                         )
                         .unwrap();
                     }
@@ -356,11 +356,11 @@ pub fn place_to_string_for_capture<'tcx>(tcx: TyCtxt<'tcx>, place: &HirPlace<'tc
                     curr_string = format!(
                         "{}.{}",
                         curr_string,
-                        def.variant(variant).fields[idx as usize].name.as_str()
+                        def.variant(variant).fields[idx].name.as_str()
                     );
                 }
                 ty::Tuple(_) => {
-                    curr_string = format!("{}.{}", curr_string, idx);
+                    curr_string = format!("{}.{}", curr_string, idx.index());
                 }
                 _ => {
                     bug!(
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index 3ce80e06a..8ef4a46a7 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -511,8 +511,6 @@ macro_rules! implement_ty_decoder {
                     read_isize -> isize;
 
                     read_bool -> bool;
-                    read_f64 -> f64;
-                    read_f32 -> f32;
                     read_char -> char;
                     read_str -> &str;
                 }
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index 527ec9f6e..bcedae233 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -1,10 +1,10 @@
 use crate::middle::resolve_bound_vars as rbv;
 use crate::mir::interpret::LitToConstInput;
-use crate::ty::{self, DefIdTree, InternalSubsts, ParamEnv, ParamEnvAnd, Ty, TyCtxt};
+use crate::ty::{self, InternalSubsts, ParamEnv, ParamEnvAnd, Ty, TyCtxt};
 use rustc_data_structures::intern::Interned;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::def_id::LocalDefId;
 use rustc_macros::HashStable;
 use std::fmt;
 
@@ -83,7 +83,7 @@ impl<'tcx> Const<'tcx> {
             None => tcx.mk_const(
                 ty::UnevaluatedConst {
                     def: def.to_global(),
-                    substs: InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
+                    substs: InternalSubsts::identity_for_item(tcx, def.did),
                 },
                 ty,
             ),
@@ -135,6 +135,9 @@ impl<'tcx> Const<'tcx> {
                 _,
                 &hir::Path { res: Res::Def(DefKind::ConstParam, def_id), .. },
             )) => {
+                // Use the type from the param's definition, since we can resolve it,
+                // not the expected parameter type from WithOptConstParam.
+                let param_ty = tcx.type_of(def_id).subst_identity();
                 match tcx.named_bound_var(expr.hir_id) {
                     Some(rbv::ResolvedArg::EarlyBound(_)) => {
                         // Find the name and index of the const parameter by indexing the generics of
@@ -143,14 +146,14 @@ impl<'tcx> Const<'tcx> {
                         let generics = tcx.generics_of(item_def_id);
                         let index = generics.param_def_id_to_index[&def_id];
                         let name = tcx.item_name(def_id);
-                        Some(tcx.mk_const(ty::ParamConst::new(index, name), ty))
+                        Some(tcx.mk_const(ty::ParamConst::new(index, name), param_ty))
                     }
                     Some(rbv::ResolvedArg::LateBound(debruijn, index, _)) => Some(tcx.mk_const(
                         ty::ConstKind::Bound(debruijn, ty::BoundVar::from_u32(index)),
-                        ty,
+                        param_ty,
                     )),
                     Some(rbv::ResolvedArg::Error(guar)) => {
-                        Some(tcx.const_error_with_guaranteed(ty, guar))
+                        Some(tcx.const_error_with_guaranteed(param_ty, guar))
                     }
                     arg => bug!("unexpected bound var resolution for {:?}: {arg:?}", expr.hir_id),
                 }
@@ -262,8 +265,8 @@ impl<'tcx> Const<'tcx> {
     }
 }
 
-pub fn const_param_default(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Const<'_>> {
-    let default_def_id = match tcx.hir().get_by_def_id(def_id.expect_local()) {
+pub fn const_param_default(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<Const<'_>> {
+    let default_def_id = match tcx.hir().get_by_def_id(def_id) {
         hir::Node::GenericParam(hir::GenericParam {
             kind: hir::GenericParamKind::Const { default: Some(ac), .. },
             ..
diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs
index eecd78ab6..c0e557d48 100644
--- a/compiler/rustc_middle/src/ty/consts/int.rs
+++ b/compiler/rustc_middle/src/ty/consts/int.rs
@@ -237,7 +237,7 @@ impl ScalarInt {
     }
 
     /// Tries to convert the `ScalarInt` to an unsigned integer of the given size.
-    /// Fails if the size of the `ScalarInt` is unequal to `size` and returns the
+    /// Fails if the size of the `ScalarInt` is not equal to `size` and returns the
     /// `ScalarInt`s size in that case.
     #[inline]
     pub fn try_to_uint(self, size: Size) -> Result<u128, Size> {
@@ -297,7 +297,7 @@ impl ScalarInt {
     }
 
     /// Tries to convert the `ScalarInt` to a signed integer of the given size.
-    /// Fails if the size of the `ScalarInt` is unequal to `size` and returns the
+    /// Fails if the size of the `ScalarInt` is not equal to `size` and returns the
     /// `ScalarInt`s size in that case.
     #[inline]
     pub fn try_to_int(self, size: Size) -> Result<i128, Size> {
@@ -306,38 +306,38 @@ impl ScalarInt {
     }
 
     /// Tries to convert the `ScalarInt` to i8.
-    /// Fails if the size of the `ScalarInt` is unequal to `Size { raw: 1 }`
+    /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 1 }`
     /// and returns the `ScalarInt`s size in that case.
     pub fn try_to_i8(self) -> Result<i8, Size> {
         self.try_to_int(Size::from_bits(8)).map(|v| i8::try_from(v).unwrap())
     }
 
     /// Tries to convert the `ScalarInt` to i16.
-    /// Fails if the size of the `ScalarInt` is unequal to `Size { raw: 2 }`
+    /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 2 }`
     /// and returns the `ScalarInt`s size in that case.
     pub fn try_to_i16(self) -> Result<i16, Size> {
         self.try_to_int(Size::from_bits(16)).map(|v| i16::try_from(v).unwrap())
     }
 
     /// Tries to convert the `ScalarInt` to i32.
-    /// Fails if the size of the `ScalarInt` is unequal to `Size { raw: 4 }`
+    /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 4 }`
     /// and returns the `ScalarInt`s size in that case.
     pub fn try_to_i32(self) -> Result<i32, Size> {
         self.try_to_int(Size::from_bits(32)).map(|v| i32::try_from(v).unwrap())
     }
 
     /// Tries to convert the `ScalarInt` to i64.
-    /// Fails if the size of the `ScalarInt` is unequal to `Size { raw: 8 }`
+    /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 8 }`
     /// and returns the `ScalarInt`s size in that case.
     pub fn try_to_i64(self) -> Result<i64, Size> {
         self.try_to_int(Size::from_bits(64)).map(|v| i64::try_from(v).unwrap())
     }
 
     /// Tries to convert the `ScalarInt` to i128.
-    /// Fails if the size of the `ScalarInt` is unequal to `Size { raw: 16 }`
+    /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 16 }`
     /// and returns the `ScalarInt`s size in that case.
     pub fn try_to_i128(self) -> Result<i128, Size> {
-        self.try_to_int(Size::from_bits(128)).map(|v| i128::try_from(v).unwrap())
+        self.try_to_int(Size::from_bits(128))
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs
index 5ed4af2e9..8b96864dd 100644
--- a/compiler/rustc_middle/src/ty/consts/valtree.rs
+++ b/compiler/rustc_middle/src/ty/consts/valtree.rs
@@ -79,7 +79,7 @@ impl<'tcx> ValTree<'tcx> {
     }
 
     pub fn try_to_target_usize(self, tcx: TyCtxt<'tcx>) -> Option<u64> {
-        self.try_to_scalar_int().map(|s| s.try_to_target_usize(tcx).ok()).flatten()
+        self.try_to_scalar_int().and_then(|s| s.try_to_target_usize(tcx).ok())
     }
 
     /// Get the values inside the ValTree as a slice of bytes. This only works for
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index d9af2fd74..63f7cc2ee 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -8,26 +8,26 @@ use crate::arena::Arena;
 use crate::dep_graph::{DepGraph, DepKindStruct};
 use crate::infer::canonical::CanonicalVarInfo;
 use crate::lint::struct_lint_level;
+use crate::metadata::ModChild;
 use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
 use crate::middle::resolve_bound_vars;
 use crate::middle::stability;
 use crate::mir::interpret::{self, Allocation, ConstAllocation};
-use crate::mir::{
-    Body, BorrowCheckResult, Field, Local, Place, PlaceElem, ProjectionKind, Promoted,
-};
+use crate::mir::{Body, BorrowCheckResult, Local, Place, PlaceElem, ProjectionKind, Promoted};
+use crate::query::LocalCrate;
 use crate::thir::Thir;
 use crate::traits;
+use crate::traits::solve;
 use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData};
 use crate::ty::query::{self, TyCtxtAt};
 use crate::ty::{
-    self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, DefIdTree, FloatTy, FloatVar,
-    FloatVid, GenericParamDefKind, ImplPolarity, InferTy, IntTy, IntVar, IntVid, List, ParamConst,
-    ParamTy, PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind, Region, RegionKind,
-    ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVar, TyVid, TypeAndMut, TypeckResults, UintTy,
-    Visibility,
+    self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, FloatTy, FloatVar, FloatVid,
+    GenericParamDefKind, ImplPolarity, InferTy, IntTy, IntVar, IntVid, List, ParamConst, ParamTy,
+    PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind, Region, RegionKind, ReprOptions,
+    TraitObjectVisitor, Ty, TyKind, TyVar, TyVid, TypeAndMut, TypeckResults, UintTy, Visibility,
 };
 use crate::ty::{GenericArg, InternalSubsts, SubstsRef};
-use rustc_ast as ast;
+use rustc_ast::{self as ast, attr};
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::intern::Interned;
@@ -37,6 +37,7 @@ use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{self, Lock, Lrc, MappedReadGuard, ReadGuard, WorkerLocal};
+use rustc_data_structures::unord::UnordSet;
 use rustc_errors::{
     DecorateLint, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed, MultiSpan,
 };
@@ -63,13 +64,14 @@ use rustc_span::def_id::{DefPathHash, StableCrateId};
 use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::{Layout, LayoutS, TargetDataLayout, VariantIdx};
+use rustc_target::abi::{FieldIdx, Layout, LayoutS, TargetDataLayout, VariantIdx};
 use rustc_target::spec::abi;
 use rustc_type_ir::sty::TyKind::*;
 use rustc_type_ir::WithCachedTypeInfo;
 use rustc_type_ir::{CollectAndApply, DynKind, Interner, TypeFlags};
 
 use std::any::Any;
+use std::assert_matches::debug_assert_matches;
 use std::borrow::Borrow;
 use std::cmp::Ordering;
 use std::fmt;
@@ -310,7 +312,7 @@ pub struct CommonLifetimes<'tcx> {
     pub re_vars: Vec<Region<'tcx>>,
 
     /// Pre-interned values of the form:
-    /// `ReLateBound(DebruijnIndex(i), BoundRegion { var: v, kind: BrAnon(v, None) })`
+    /// `ReLateBound(DebruijnIndex(i), BoundRegion { var: v, kind: BrAnon(None) })`
     /// for small values of `i` and `v`.
     pub re_late_bounds: Vec<Vec<Region<'tcx>>>,
 }
@@ -385,10 +387,7 @@ impl<'tcx> CommonLifetimes<'tcx> {
                     .map(|v| {
                         mk(ty::ReLateBound(
                             ty::DebruijnIndex::from(i),
-                            ty::BoundRegion {
-                                var: ty::BoundVar::from(v),
-                                kind: ty::BrAnon(v, None),
-                            },
+                            ty::BoundRegion { var: ty::BoundVar::from(v), kind: ty::BrAnon(None) },
                         ))
                     })
                     .collect()
@@ -537,6 +536,9 @@ pub struct GlobalCtxt<'tcx> {
     /// Merge this with `selection_cache`?
     pub evaluation_cache: traits::EvaluationCache<'tcx>,
 
+    /// Caches the results of goal evaluation in the new solver.
+    pub new_solver_evaluation_cache: solve::EvaluationCache<'tcx>,
+
     /// Data layout specification for the current target.
     pub data_layout: TargetDataLayout,
 
@@ -712,6 +714,7 @@ impl<'tcx> TyCtxt<'tcx> {
             pred_rcache: Default::default(),
             selection_cache: Default::default(),
             evaluation_cache: Default::default(),
+            new_solver_evaluation_cache: Default::default(),
             data_layout,
             alloc_map: Lock::new(interpret::AllocMap::new()),
         }
@@ -922,7 +925,7 @@ impl<'tcx> TyCtxt<'tcx> {
             crate_name,
             // Don't print the whole stable crate id. That's just
             // annoying in debug output.
-            stable_crate_id.to_u64() >> 8 * 6,
+            stable_crate_id.to_u64() >> (8 * 6),
             self.def_path(def_id).to_string_no_crate_verbose()
         )
     }
@@ -2044,6 +2047,12 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn mk_alias(self, kind: ty::AliasKind, alias_ty: ty::AliasTy<'tcx>) -> Ty<'tcx> {
+        debug_assert_matches!(
+            (kind, self.def_kind(alias_ty.def_id)),
+            (ty::Opaque, DefKind::OpaqueTy)
+                | (ty::Projection, DefKind::AssocTy)
+                | (ty::Opaque | ty::Projection, DefKind::ImplTraitPlaceholder)
+        );
         self.mk_ty_from_kind(Alias(kind, alias_ty))
     }
 
@@ -2064,10 +2073,9 @@ impl<'tcx> TyCtxt<'tcx> {
         bound_region: ty::BoundRegion,
     ) -> Region<'tcx> {
         // Use a pre-interned one when possible.
-        if let ty::BoundRegion { var, kind: ty::BrAnon(v, None) } = bound_region
-            && var.as_u32() == v
+        if let ty::BoundRegion { var, kind: ty::BrAnon(None) } = bound_region
             && let Some(inner) = self.lifetimes.re_late_bounds.get(debruijn.as_usize())
-            && let Some(re) = inner.get(v as usize).copied()
+            && let Some(re) = inner.get(var.as_usize()).copied()
         {
             re
         } else {
@@ -2112,7 +2120,7 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    pub fn mk_place_field(self, place: Place<'tcx>, f: Field, ty: Ty<'tcx>) -> Place<'tcx> {
+    pub fn mk_place_field(self, place: Place<'tcx>, f: FieldIdx, ty: Ty<'tcx>) -> Place<'tcx> {
         self.mk_place_elem(place, PlaceElem::Field(f, ty))
     }
 
@@ -2372,7 +2380,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn in_scope_traits(self, id: HirId) -> Option<&'tcx [TraitCandidate]> {
         let map = self.in_scope_traits_map(id.owner)?;
         let candidates = map.get(&id.local_id)?;
-        Some(&*candidates)
+        Some(candidates)
     }
 
     pub fn named_bound_var(self, id: HirId) -> Option<resolve_bound_vars::ResolvedArg> {
@@ -2440,6 +2448,40 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn trait_solver_next(self) -> bool {
         self.sess.opts.unstable_opts.trait_solver == rustc_session::config::TraitSolver::Next
     }
+
+    pub fn lower_impl_trait_in_trait_to_assoc_ty(self) -> bool {
+        self.sess.opts.unstable_opts.lower_impl_trait_in_trait_to_assoc_ty
+    }
+
+    pub fn is_impl_trait_in_trait(self, def_id: DefId) -> bool {
+        if self.lower_impl_trait_in_trait_to_assoc_ty() {
+            self.opt_rpitit_info(def_id).is_some()
+        } else {
+            self.def_kind(def_id) == DefKind::ImplTraitPlaceholder
+        }
+    }
+
+    /// Named module children from all items except `use` and `extern crate` imports.
+    ///
+    /// In addition to regular items this list also includes struct or variant constructors, and
+    /// items inside `extern {}` blocks because all of them introduce names into parent module.
+    /// For non-reexported children every such name is associated with a separate `DefId`.
+    ///
+    /// Module here is understood in name resolution sense - it can be a `mod` item,
+    /// or a crate root, or an enum, or a trait.
+    pub fn module_children_non_reexports(self, def_id: LocalDefId) -> &'tcx [LocalDefId] {
+        self.resolutions(()).module_children_non_reexports.get(&def_id).map_or(&[], |v| &v[..])
+    }
+
+    /// Named module children from `use` and `extern crate` imports.
+    ///
+    /// Reexported names are not associated with individual `DefId`s,
+    /// e.g. a glob import can introduce a lot of names, all with the same `DefId`.
+    /// That's why the list needs to contain `ModChild` structures describing all the names
+    /// individually instead of `DefId`s.
+    pub fn module_children_reexports(self, def_id: LocalDefId) -> &'tcx [ModChild] {
+        self.resolutions(()).module_children_reexports.get(&def_id).map_or(&[], |v| &v[..])
+    }
 }
 
 impl<'tcx> TyCtxtAt<'tcx> {
@@ -2482,26 +2524,21 @@ pub struct DeducedParamAttrs {
 }
 
 pub fn provide(providers: &mut ty::query::Providers) {
-    providers.module_reexports =
-        |tcx, id| tcx.resolutions(()).reexport_map.get(&id).map(|v| &v[..]);
     providers.maybe_unused_trait_imports =
         |tcx, ()| &tcx.resolutions(()).maybe_unused_trait_imports;
     providers.names_imported_by_glob_use = |tcx, id| {
-        tcx.arena.alloc(tcx.resolutions(()).glob_map.get(&id).cloned().unwrap_or_default())
+        tcx.arena.alloc(UnordSet::from(
+            tcx.resolutions(()).glob_map.get(&id).cloned().unwrap_or_default(),
+        ))
     };
 
     providers.extern_mod_stmt_cnum =
         |tcx, id| tcx.resolutions(()).extern_crate_map.get(&id).cloned();
-    providers.is_panic_runtime = |tcx, cnum| {
-        assert_eq!(cnum, LOCAL_CRATE);
-        tcx.sess.contains_name(tcx.hir().krate_attrs(), sym::panic_runtime)
-    };
-    providers.is_compiler_builtins = |tcx, cnum| {
-        assert_eq!(cnum, LOCAL_CRATE);
-        tcx.sess.contains_name(tcx.hir().krate_attrs(), sym::compiler_builtins)
-    };
-    providers.has_panic_handler = |tcx, cnum| {
-        assert_eq!(cnum, LOCAL_CRATE);
+    providers.is_panic_runtime =
+        |tcx, LocalCrate| attr::contains_name(tcx.hir().krate_attrs(), sym::panic_runtime);
+    providers.is_compiler_builtins =
+        |tcx, LocalCrate| attr::contains_name(tcx.hir().krate_attrs(), sym::compiler_builtins);
+    providers.has_panic_handler = |tcx, LocalCrate| {
         // We want to check if the panic handler was defined in this crate
         tcx.lang_items().panic_impl().map_or(false, |did| did.is_local())
     };
diff --git a/compiler/rustc_middle/src/ty/context/tls.rs b/compiler/rustc_middle/src/ty/context/tls.rs
index 5426ac8d7..fb0d90930 100644
--- a/compiler/rustc_middle/src/ty/context/tls.rs
+++ b/compiler/rustc_middle/src/ty/context/tls.rs
@@ -4,6 +4,8 @@ use crate::dep_graph::TaskDepsRef;
 use crate::ty::query;
 use rustc_data_structures::sync::{self, Lock};
 use rustc_errors::Diagnostic;
+#[cfg(not(parallel_compiler))]
+use std::cell::Cell;
 use std::mem;
 use std::ptr;
 use thin_vec::ThinVec;
@@ -47,52 +49,15 @@ impl<'a, 'tcx> ImplicitCtxt<'a, 'tcx> {
     }
 }
 
+// Import the thread-local variable from Rayon, which is preserved for Rayon jobs.
 #[cfg(parallel_compiler)]
-mod tlv {
-    use rustc_rayon_core as rayon_core;
-    use std::ptr;
-
-    /// Gets Rayon's thread-local variable, which is preserved for Rayon jobs.
-    /// This is used to get the pointer to the current `ImplicitCtxt`.
-    #[inline]
-    pub(super) fn get_tlv() -> *const () {
-        ptr::from_exposed_addr(rayon_core::tlv::get())
-    }
-
-    /// Sets Rayon's thread-local variable, which is preserved for Rayon jobs
-    /// to `value` during the call to `f`. It is restored to its previous value after.
-    /// This is used to set the pointer to the new `ImplicitCtxt`.
-    #[inline]
-    pub(super) fn with_tlv<F: FnOnce() -> R, R>(value: *const (), f: F) -> R {
-        rayon_core::tlv::with(value.expose_addr(), f)
-    }
-}
+use rayon_core::tlv::TLV;
 
+// Otherwise define our own
 #[cfg(not(parallel_compiler))]
-mod tlv {
-    use std::cell::Cell;
-    use std::ptr;
-
-    thread_local! {
-        /// A thread local variable that stores a pointer to the current `ImplicitCtxt`.
-        static TLV: Cell<*const ()> = const { Cell::new(ptr::null()) };
-    }
-
-    /// Gets the pointer to the current `ImplicitCtxt`.
-    #[inline]
-    pub(super) fn get_tlv() -> *const () {
-        TLV.with(|tlv| tlv.get())
-    }
-
-    /// Sets TLV to `value` during the call to `f`.
-    /// It is restored to its previous value after.
-    /// This is used to set the pointer to the new `ImplicitCtxt`.
-    #[inline]
-    pub(super) fn with_tlv<F: FnOnce() -> R, R>(value: *const (), f: F) -> R {
-        let old = TLV.replace(value);
-        let _reset = rustc_data_structures::OnDrop(move || TLV.set(old));
-        f()
-    }
+thread_local! {
+    /// A thread local variable that stores a pointer to the current `ImplicitCtxt`.
+    static TLV: Cell<*const ()> = const { Cell::new(ptr::null()) };
 }
 
 #[inline]
@@ -111,7 +76,11 @@ pub fn enter_context<'a, 'tcx, F, R>(context: &ImplicitCtxt<'a, 'tcx>, f: F) ->
 where
     F: FnOnce() -> R,
 {
-    tlv::with_tlv(erase(context), f)
+    TLV.with(|tlv| {
+        let old = tlv.replace(erase(context));
+        let _reset = rustc_data_structures::OnDrop(move || tlv.set(old));
+        f()
+    })
 }
 
 /// Allows access to the current `ImplicitCtxt` in a closure if one is available.
@@ -120,7 +89,7 @@ pub fn with_context_opt<F, R>(f: F) -> R
 where
     F: for<'a, 'tcx> FnOnce(Option<&ImplicitCtxt<'a, 'tcx>>) -> R,
 {
-    let context = tlv::get_tlv();
+    let context = TLV.get();
     if context.is_null() {
         f(None)
     } else {
diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs
index e894e1aaf..ae0bb4949 100644
--- a/compiler/rustc_middle/src/ty/diagnostics.rs
+++ b/compiler/rustc_middle/src/ty/diagnostics.rs
@@ -3,9 +3,9 @@
 use std::ops::ControlFlow;
 
 use crate::ty::{
-    AliasTy, Const, ConstKind, DefIdTree, FallibleTypeFolder, InferConst, InferTy, Opaque,
-    PolyTraitPredicate, Projection, Ty, TyCtxt, TypeFoldable, TypeSuperFoldable,
-    TypeSuperVisitable, TypeVisitable, TypeVisitor,
+    AliasTy, Const, ConstKind, FallibleTypeFolder, InferConst, InferTy, Opaque, PolyTraitPredicate,
+    Projection, Ty, TyCtxt, TypeFoldable, TypeSuperFoldable, TypeSuperVisitable, TypeVisitable,
+    TypeVisitor,
 };
 
 use rustc_data_structures::fx::FxHashMap;
@@ -117,7 +117,7 @@ pub fn suggest_arbitrary_trait_bound<'tcx>(
     }
 
     let param_name = trait_pred.skip_binder().self_ty().to_string();
-    let mut constraint = trait_pred.print_modifiers_and_trait_path().to_string();
+    let mut constraint = trait_pred.to_string();
 
     if let Some((name, term)) = associated_ty {
         // FIXME: this case overlaps with code in TyCtxt::note_and_explain_type_err.
@@ -144,7 +144,7 @@ pub fn suggest_arbitrary_trait_bound<'tcx>(
              this requirement",
             if generics.where_clause_span.is_empty() { "introducing a" } else { "extending the" },
         ),
-        format!("{} {}: {}", generics.add_where_or_trailing_comma(), param_name, constraint),
+        format!("{} {constraint}", generics.add_where_or_trailing_comma()),
         Applicability::MaybeIncorrect,
     );
     true
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index 9c171a69d..aff6c77e0 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -151,12 +151,8 @@ impl<'tcx> TypeError<'tcx> {
             .into(),
             RegionsPlaceholderMismatch => "one type is more general than the other".into(),
             ArgumentSorts(values, _) | Sorts(values) => {
-                let mut expected = values.expected.sort_string(tcx);
-                let mut found = values.found.sort_string(tcx);
-                if expected == found {
-                    expected = values.expected.sort_string(tcx);
-                    found = values.found.sort_string(tcx);
-                }
+                let expected = values.expected.sort_string(tcx);
+                let found = values.found.sort_string(tcx);
                 report_maybe_different(&expected, &found).into()
             }
             Traits(values) => {
diff --git a/compiler/rustc_middle/src/ty/fast_reject.rs b/compiler/rustc_middle/src/ty/fast_reject.rs
index 59deade0a..31d00b65e 100644
--- a/compiler/rustc_middle/src/ty/fast_reject.rs
+++ b/compiler/rustc_middle/src/ty/fast_reject.rs
@@ -1,6 +1,6 @@
 use crate::mir::Mutability;
 use crate::ty::subst::GenericArgKind;
-use crate::ty::{self, Ty, TyCtxt, TypeVisitableExt};
+use crate::ty::{self, SubstsRef, Ty, TyCtxt, TypeVisitableExt};
 use rustc_hir::def_id::DefId;
 use std::fmt::Debug;
 use std::hash::Hash;
@@ -51,15 +51,36 @@ pub enum SimplifiedType {
 /// generic parameters as if they were inference variables in that case.
 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
 pub enum TreatParams {
-    /// Treat parameters as placeholders in the given environment.
+    /// Treat parameters as infer vars. This is the correct mode for caching
+    /// an impl's type for lookup.
+    AsCandidateKey,
+    /// Treat parameters as placeholders in the given environment. This is the
+    /// correct mode for *lookup*, as during candidate selection.
     ///
-    /// Note that this also causes us to treat projections as if they were
-    /// placeholders. This is only correct if the given projection cannot
-    /// be normalized in the current context. Even if normalization fails,
-    /// it may still succeed later if the projection contains any inference
-    /// variables.
-    AsPlaceholder,
-    AsInfer,
+    /// This also treats projections with inference variables as infer vars
+    /// since they could be further normalized.
+    ForLookup,
+    /// Treat parameters as placeholders in the given environment. This is the
+    /// correct mode for *lookup*, as during candidate selection.
+    ///
+    /// N.B. during deep rejection, this acts identically to `ForLookup`.
+    NextSolverLookup,
+}
+
+/// During fast-rejection, we have the choice of treating projection types
+/// as either simplifyable or not, depending on whether we expect the projection
+/// to be normalized/rigid.
+#[derive(PartialEq, Eq, Debug, Clone, Copy)]
+pub enum TreatProjections {
+    /// In the old solver we don't try to normalize projections
+    /// when looking up impls and only access them by using the
+    /// current self type. This means that if the self type is
+    /// a projection which could later be normalized, we must not
+    /// treat it as rigid.
+    ForLookup,
+    /// We can treat projections in the self type as opaque as
+    /// we separately look up impls for the normalized self type.
+    NextSolverLookup,
 }
 
 /// Tries to simplify a type by only returning the outermost injective¹ layer, if one exists.
@@ -115,19 +136,20 @@ pub fn simplify_type<'tcx>(
         ty::FnPtr(f) => Some(FunctionSimplifiedType(f.skip_binder().inputs().len())),
         ty::Placeholder(..) => Some(PlaceholderSimplifiedType),
         ty::Param(_) => match treat_params {
-            TreatParams::AsPlaceholder => Some(PlaceholderSimplifiedType),
-            TreatParams::AsInfer => None,
+            TreatParams::ForLookup | TreatParams::NextSolverLookup => {
+                Some(PlaceholderSimplifiedType)
+            }
+            TreatParams::AsCandidateKey => None,
         },
         ty::Alias(..) => match treat_params {
             // When treating `ty::Param` as a placeholder, projections also
             // don't unify with anything else as long as they are fully normalized.
             //
             // We will have to be careful with lazy normalization here.
-            TreatParams::AsPlaceholder if !ty.has_non_region_infer() => {
-                debug!("treating `{}` as a placeholder", ty);
-                Some(PlaceholderSimplifiedType)
-            }
-            TreatParams::AsPlaceholder | TreatParams::AsInfer => None,
+            // FIXME(lazy_normalization): This is probably not right...
+            TreatParams::ForLookup if !ty.has_non_region_infer() => Some(PlaceholderSimplifiedType),
+            TreatParams::NextSolverLookup => Some(PlaceholderSimplifiedType),
+            TreatParams::ForLookup | TreatParams::AsCandidateKey => None,
         },
         ty::Foreign(def_id) => Some(ForeignSimplifiedType(def_id)),
         ty::Bound(..) | ty::Infer(_) | ty::Error(_) => None,
@@ -166,22 +188,24 @@ pub struct DeepRejectCtxt {
 }
 
 impl DeepRejectCtxt {
-    pub fn generic_args_may_unify<'tcx>(
+    pub fn substs_refs_may_unify<'tcx>(
         self,
-        obligation_arg: ty::GenericArg<'tcx>,
-        impl_arg: ty::GenericArg<'tcx>,
+        obligation_substs: SubstsRef<'tcx>,
+        impl_substs: SubstsRef<'tcx>,
     ) -> bool {
-        match (obligation_arg.unpack(), impl_arg.unpack()) {
-            // We don't fast reject based on regions for now.
-            (GenericArgKind::Lifetime(_), GenericArgKind::Lifetime(_)) => true,
-            (GenericArgKind::Type(obl), GenericArgKind::Type(imp)) => {
-                self.types_may_unify(obl, imp)
-            }
-            (GenericArgKind::Const(obl), GenericArgKind::Const(imp)) => {
-                self.consts_may_unify(obl, imp)
+        iter::zip(obligation_substs, impl_substs).all(|(obl, imp)| {
+            match (obl.unpack(), imp.unpack()) {
+                // We don't fast reject based on regions for now.
+                (GenericArgKind::Lifetime(_), GenericArgKind::Lifetime(_)) => true,
+                (GenericArgKind::Type(obl), GenericArgKind::Type(imp)) => {
+                    self.types_may_unify(obl, imp)
+                }
+                (GenericArgKind::Const(obl), GenericArgKind::Const(imp)) => {
+                    self.consts_may_unify(obl, imp)
+                }
+                _ => bug!("kind mismatch: {obl} {imp}"),
             }
-            _ => bug!("kind mismatch: {obligation_arg} {impl_arg}"),
-        }
+        })
     }
 
     pub fn types_may_unify<'tcx>(self, obligation_ty: Ty<'tcx>, impl_ty: Ty<'tcx>) -> bool {
@@ -236,9 +260,7 @@ impl DeepRejectCtxt {
             },
             ty::Adt(obl_def, obl_substs) => match k {
                 &ty::Adt(impl_def, impl_substs) => {
-                    obl_def == impl_def
-                        && iter::zip(obl_substs, impl_substs)
-                            .all(|(obl, imp)| self.generic_args_may_unify(obl, imp))
+                    obl_def == impl_def && self.substs_refs_may_unify(obl_substs, impl_substs)
                 }
                 _ => false,
             },
@@ -290,15 +312,20 @@ impl DeepRejectCtxt {
             // Impls cannot contain these types as these cannot be named directly.
             ty::FnDef(..) | ty::Closure(..) | ty::Generator(..) => false,
 
+            // Placeholder types don't unify with anything on their own
             ty::Placeholder(..) | ty::Bound(..) => false,
 
             // Depending on the value of `treat_obligation_params`, we either
             // treat generic parameters like placeholders or like inference variables.
             ty::Param(_) => match self.treat_obligation_params {
-                TreatParams::AsPlaceholder => false,
-                TreatParams::AsInfer => true,
+                TreatParams::ForLookup | TreatParams::NextSolverLookup => false,
+                TreatParams::AsCandidateKey => true,
             },
 
+            ty::Infer(ty::IntVar(_)) => impl_ty.is_integral(),
+
+            ty::Infer(ty::FloatVar(_)) => impl_ty.is_floating_point(),
+
             ty::Infer(_) => true,
 
             // As we're walking the whole type, it may encounter projections
@@ -333,10 +360,13 @@ impl DeepRejectCtxt {
         let k = impl_ct.kind();
         match obligation_ct.kind() {
             ty::ConstKind::Param(_) => match self.treat_obligation_params {
-                TreatParams::AsPlaceholder => false,
-                TreatParams::AsInfer => true,
+                TreatParams::ForLookup | TreatParams::NextSolverLookup => false,
+                TreatParams::AsCandidateKey => true,
             },
 
+            // Placeholder consts don't unify with anything on their own
+            ty::ConstKind::Placeholder(_) => false,
+
             // As we don't necessarily eagerly evaluate constants,
             // they might unify with any value.
             ty::ConstKind::Expr(_) | ty::ConstKind::Unevaluated(_) | ty::ConstKind::Error(_) => {
@@ -349,7 +379,7 @@ impl DeepRejectCtxt {
 
             ty::ConstKind::Infer(_) => true,
 
-            ty::ConstKind::Bound(..) | ty::ConstKind::Placeholder(_) => {
+            ty::ConstKind::Bound(..) => {
                 bug!("unexpected obl const: {:?}", obligation_ct)
             }
         }
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index 91241ff40..5a6ee1238 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -288,7 +288,7 @@ impl FlagComputation {
                 self.add_ty(ty);
             }
             ty::PredicateKind::Ambiguous => {}
-            ty::PredicateKind::AliasEq(t1, t2) => {
+            ty::PredicateKind::AliasRelate(t1, t2, _) => {
                 self.add_term(t1);
                 self.add_term(t2);
             }
diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs
index d66f436f9..203e16bea 100644
--- a/compiler/rustc_middle/src/ty/fold.rs
+++ b/compiler/rustc_middle/src/ty/fold.rs
@@ -51,9 +51,7 @@ where
 // Region folder
 
 impl<'tcx> TyCtxt<'tcx> {
-    /// Folds the escaping and free regions in `value` using `f`, and
-    /// sets `skipped_regions` to true if any late-bound region was found
-    /// and skipped.
+    /// Folds the escaping and free regions in `value` using `f`.
     pub fn fold_regions<T>(
         self,
         value: T,
@@ -64,17 +62,6 @@ impl<'tcx> TyCtxt<'tcx> {
     {
         value.fold_with(&mut RegionFolder::new(self, &mut f))
     }
-
-    pub fn super_fold_regions<T>(
-        self,
-        value: T,
-        mut f: impl FnMut(ty::Region<'tcx>, ty::DebruijnIndex) -> ty::Region<'tcx>,
-    ) -> T
-    where
-        T: TypeSuperFoldable<TyCtxt<'tcx>>,
-    {
-        value.super_fold_with(&mut RegionFolder::new(self, &mut f))
-    }
 }
 
 /// Folds over the substructure of a type, visiting its component
@@ -392,9 +379,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 let index = entry.index();
                 let var = ty::BoundVar::from_usize(index);
                 let kind = entry
-                    .or_insert_with(|| {
-                        ty::BoundVariableKind::Region(ty::BrAnon(index as u32, None))
-                    })
+                    .or_insert_with(|| ty::BoundVariableKind::Region(ty::BrAnon(None)))
                     .expect_region();
                 let br = ty::BoundRegion { var, kind };
                 self.tcx.mk_re_late_bound(ty::INNERMOST, br)
@@ -404,9 +389,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 let index = entry.index();
                 let var = ty::BoundVar::from_usize(index);
                 let kind = entry
-                    .or_insert_with(|| {
-                        ty::BoundVariableKind::Ty(ty::BoundTyKind::Anon(index as u32))
-                    })
+                    .or_insert_with(|| ty::BoundVariableKind::Ty(ty::BoundTyKind::Anon))
                     .expect_ty();
                 self.tcx.mk_bound(ty::INNERMOST, BoundTy { var, kind })
             }
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
index e268553f8..ac42d6e05 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
@@ -1,5 +1,5 @@
 use crate::ty::context::TyCtxt;
-use crate::ty::{self, DefId, DefIdTree, ParamEnv, Ty};
+use crate::ty::{self, DefId, ParamEnv, Ty};
 
 /// Represents whether some type is inhabited in a given context.
 /// Examples of uninhabited types are `!`, `enum Void {}`, or a struct
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index f4028a5a9..e73225f70 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -82,6 +82,11 @@ pub enum InstanceDef<'tcx> {
     /// The `DefId` is the ID of the `call_once` method in `FnOnce`.
     ClosureOnceShim { call_once: DefId, track_caller: bool },
 
+    /// Compiler-generated accessor for thread locals which returns a reference to the thread local
+    /// the `DefId` defines. This is used to export thread locals from dylibs on platforms lacking
+    /// native support.
+    ThreadLocalShim(DefId),
+
     /// `core::ptr::drop_in_place::<T>`.
     ///
     /// The `DefId` is for `core::ptr::drop_in_place`.
@@ -96,6 +101,13 @@ pub enum InstanceDef<'tcx> {
     ///
     /// The `DefId` is for `Clone::clone`, the `Ty` is the type `T` with the builtin `Clone` impl.
     CloneShim(DefId, Ty<'tcx>),
+
+    /// Compiler-generated `<T as FnPtr>::addr` implementation.
+    ///
+    /// Automatically generated for all potentially higher-ranked `fn(I) -> R` types.
+    ///
+    /// The `DefId` is for `FnPtr::addr`, the `Ty` is the type `T`.
+    FnPtrAddrShim(DefId, Ty<'tcx>),
 }
 
 impl<'tcx> Instance<'tcx> {
@@ -149,9 +161,11 @@ impl<'tcx> InstanceDef<'tcx> {
             | InstanceDef::FnPtrShim(def_id, _)
             | InstanceDef::Virtual(def_id, _)
             | InstanceDef::Intrinsic(def_id)
+            | InstanceDef::ThreadLocalShim(def_id)
             | InstanceDef::ClosureOnceShim { call_once: def_id, track_caller: _ }
             | InstanceDef::DropGlue(def_id, _)
-            | InstanceDef::CloneShim(def_id, _) => def_id,
+            | InstanceDef::CloneShim(def_id, _)
+            | InstanceDef::FnPtrAddrShim(def_id, _) => def_id,
         }
     }
 
@@ -159,7 +173,9 @@ impl<'tcx> InstanceDef<'tcx> {
     pub fn def_id_if_not_guaranteed_local_codegen(self) -> Option<DefId> {
         match self {
             ty::InstanceDef::Item(def) => Some(def.did),
-            ty::InstanceDef::DropGlue(def_id, Some(_)) => Some(def_id),
+            ty::InstanceDef::DropGlue(def_id, Some(_)) | InstanceDef::ThreadLocalShim(def_id) => {
+                Some(def_id)
+            }
             InstanceDef::VTableShim(..)
             | InstanceDef::ReifyShim(..)
             | InstanceDef::FnPtrShim(..)
@@ -167,7 +183,8 @@ impl<'tcx> InstanceDef<'tcx> {
             | InstanceDef::Intrinsic(..)
             | InstanceDef::ClosureOnceShim { .. }
             | InstanceDef::DropGlue(..)
-            | InstanceDef::CloneShim(..) => None,
+            | InstanceDef::CloneShim(..)
+            | InstanceDef::FnPtrAddrShim(..) => None,
         }
     }
 
@@ -182,12 +199,18 @@ impl<'tcx> InstanceDef<'tcx> {
             | InstanceDef::Intrinsic(def_id)
             | InstanceDef::ClosureOnceShim { call_once: def_id, track_caller: _ }
             | InstanceDef::DropGlue(def_id, _)
-            | InstanceDef::CloneShim(def_id, _) => ty::WithOptConstParam::unknown(def_id),
+            | InstanceDef::CloneShim(def_id, _)
+            | InstanceDef::ThreadLocalShim(def_id)
+            | InstanceDef::FnPtrAddrShim(def_id, _) => ty::WithOptConstParam::unknown(def_id),
         }
     }
 
     #[inline]
-    pub fn get_attrs(&self, tcx: TyCtxt<'tcx>, attr: Symbol) -> ty::Attributes<'tcx> {
+    pub fn get_attrs(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        attr: Symbol,
+    ) -> impl Iterator<Item = &'tcx rustc_ast::Attribute> {
         tcx.get_attrs(self.def_id(), attr)
     }
 
@@ -201,6 +224,7 @@ impl<'tcx> InstanceDef<'tcx> {
         let def_id = match *self {
             ty::InstanceDef::Item(def) => def.did,
             ty::InstanceDef::DropGlue(_, Some(_)) => return false,
+            ty::InstanceDef::ThreadLocalShim(_) => return false,
             _ => return true,
         };
         matches!(
@@ -241,6 +265,9 @@ impl<'tcx> InstanceDef<'tcx> {
                 )
             });
         }
+        if let ty::InstanceDef::ThreadLocalShim(..) = *self {
+            return false;
+        }
         tcx.codegen_fn_attrs(self.def_id()).requests_inline()
     }
 
@@ -264,6 +291,8 @@ impl<'tcx> InstanceDef<'tcx> {
     pub fn has_polymorphic_mir_body(&self) -> bool {
         match *self {
             InstanceDef::CloneShim(..)
+            | InstanceDef::ThreadLocalShim(..)
+            | InstanceDef::FnPtrAddrShim(..)
             | InstanceDef::FnPtrShim(..)
             | InstanceDef::DropGlue(_, Some(_)) => false,
             InstanceDef::ClosureOnceShim { .. }
@@ -295,6 +324,7 @@ fn fmt_instance(
         InstanceDef::Item(_) => Ok(()),
         InstanceDef::VTableShim(_) => write!(f, " - shim(vtable)"),
         InstanceDef::ReifyShim(_) => write!(f, " - shim(reify)"),
+        InstanceDef::ThreadLocalShim(_) => write!(f, " - shim(tls)"),
         InstanceDef::Intrinsic(_) => write!(f, " - intrinsic"),
         InstanceDef::Virtual(_, num) => write!(f, " - virtual#{}", num),
         InstanceDef::FnPtrShim(_, ty) => write!(f, " - shim({})", ty),
@@ -302,6 +332,7 @@ fn fmt_instance(
         InstanceDef::DropGlue(_, None) => write!(f, " - shim(None)"),
         InstanceDef::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({}))", ty),
         InstanceDef::CloneShim(_, ty) => write!(f, " - shim({})", ty),
+        InstanceDef::FnPtrAddrShim(_, ty) => write!(f, " - shim({})", ty),
     }
 }
 
@@ -781,6 +812,12 @@ fn needs_fn_once_adapter_shim(
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Decodable, Encodable, HashStable)]
 pub struct UnusedGenericParams(FiniteBitSet<u32>);
 
+impl Default for UnusedGenericParams {
+    fn default() -> Self {
+        UnusedGenericParams::new_all_used()
+    }
+}
+
 impl UnusedGenericParams {
     pub fn new_all_unused(amount: u32) -> Self {
         let mut bitset = FiniteBitSet::new_empty();
@@ -807,4 +844,12 @@ impl UnusedGenericParams {
     pub fn all_used(&self) -> bool {
         self.0.is_empty()
     }
+
+    pub fn bits(&self) -> u32 {
+        self.0.0
+    }
+
+    pub fn from_bits(bits: u32) -> UnusedGenericParams {
+        UnusedGenericParams(FiniteBitSet(bits))
+    }
 }
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 254ffc33c..195d951f9 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -5,7 +5,7 @@ use crate::ty::{self, ReprOptions, Ty, TyCtxt, TypeVisitableExt};
 use rustc_errors::{DiagnosticBuilder, Handler, IntoDiagnostic};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
-use rustc_index::vec::Idx;
+use rustc_index::vec::IndexVec;
 use rustc_session::config::OptLevel;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
@@ -281,6 +281,12 @@ pub enum SizeSkeleton<'tcx> {
     /// Any statically computable Layout.
     Known(Size),
 
+    /// This is a generic const expression (i.e. N * 2), which may contain some parameters.
+    /// It must be of type usize, and represents the size of a type in bytes.
+    /// It is not required to be evaluatable to a concrete value, but can be used to check
+    /// that another SizeSkeleton is of equal size.
+    Generic(ty::Const<'tcx>),
+
     /// A potentially-fat pointer.
     Pointer {
         /// If true, this pointer is never null.
@@ -326,6 +332,37 @@ impl<'tcx> SizeSkeleton<'tcx> {
                     ),
                 }
             }
+            ty::Array(inner, len)
+                if len.ty() == tcx.types.usize && tcx.features().transmute_generic_consts =>
+            {
+                match SizeSkeleton::compute(inner, tcx, param_env)? {
+                    // This may succeed because the multiplication of two types may overflow
+                    // but a single size of a nested array will not.
+                    SizeSkeleton::Known(s) => {
+                        if let Some(c) = len.try_eval_target_usize(tcx, param_env) {
+                            let size = s
+                                .bytes()
+                                .checked_mul(c)
+                                .ok_or_else(|| LayoutError::SizeOverflow(ty))?;
+                            return Ok(SizeSkeleton::Known(Size::from_bytes(size)));
+                        }
+                        let len = tcx.expand_abstract_consts(len);
+                        let prev = ty::Const::from_target_usize(tcx, s.bytes());
+                        let Some(gen_size) = mul_sorted_consts(tcx, param_env, len, prev) else {
+                            return Err(LayoutError::SizeOverflow(ty));
+                        };
+                        Ok(SizeSkeleton::Generic(gen_size))
+                    }
+                    SizeSkeleton::Pointer { .. } => Err(err),
+                    SizeSkeleton::Generic(g) => {
+                        let len = tcx.expand_abstract_consts(len);
+                        let Some(gen_size) = mul_sorted_consts(tcx, param_env, len, g) else {
+                            return Err(LayoutError::SizeOverflow(ty));
+                        };
+                        Ok(SizeSkeleton::Generic(gen_size))
+                    }
+                }
+            }
 
             ty::Adt(def, substs) => {
                 // Only newtypes and enums w/ nullable pointer optimization.
@@ -335,7 +372,7 @@ impl<'tcx> SizeSkeleton<'tcx> {
 
                 // Get a zero-sized variant or a pointer newtype.
                 let zero_or_ptr_variant = |i| {
-                    let i = VariantIdx::new(i);
+                    let i = VariantIdx::from_usize(i);
                     let fields =
                         def.variant(i).fields.iter().map(|field| {
                             SizeSkeleton::compute(field.ty(tcx, substs), tcx, param_env)
@@ -355,6 +392,9 @@ impl<'tcx> SizeSkeleton<'tcx> {
                                 }
                                 ptr = Some(field);
                             }
+                            SizeSkeleton::Generic(_) => {
+                                return Err(err);
+                            }
                         }
                     }
                     Ok(ptr)
@@ -410,11 +450,66 @@ impl<'tcx> SizeSkeleton<'tcx> {
             (SizeSkeleton::Pointer { tail: a, .. }, SizeSkeleton::Pointer { tail: b, .. }) => {
                 a == b
             }
+            // constants are always pre-normalized into a canonical form so this
+            // only needs to check if their pointers are identical.
+            (SizeSkeleton::Generic(a), SizeSkeleton::Generic(b)) => a == b,
             _ => false,
         }
     }
 }
 
+/// When creating the layout for types with abstract conts in their size (i.e. [usize; 4 * N]),
+/// to ensure that they have a canonical order and can be compared directly we combine all
+/// constants, and sort the other terms. This allows comparison of expressions of sizes,
+/// allowing for things like transmutating between types that depend on generic consts.
+/// This returns `None` if multiplication of constants overflows.
+fn mul_sorted_consts<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    a: ty::Const<'tcx>,
+    b: ty::Const<'tcx>,
+) -> Option<ty::Const<'tcx>> {
+    use crate::mir::BinOp::Mul;
+    use ty::ConstKind::Expr;
+    use ty::Expr::Binop;
+
+    let mut work = vec![a, b];
+    let mut done = vec![];
+    while let Some(n) = work.pop() {
+        if let Expr(Binop(Mul, l, r)) = n.kind() {
+            work.push(l);
+            work.push(r)
+        } else {
+            done.push(n);
+        }
+    }
+    let mut k = 1;
+    let mut overflow = false;
+    done.retain(|c| {
+        let Some(c) = c.try_eval_target_usize(tcx, param_env) else {
+            return true;
+        };
+        let Some(next) = c.checked_mul(k) else {
+            overflow = true;
+            return false;
+        };
+        k = next;
+        false
+    });
+    if overflow {
+        return None;
+    }
+    if k != 1 {
+        done.push(ty::Const::from_target_usize(tcx, k));
+    } else if k == 0 {
+        return Some(ty::Const::from_target_usize(tcx, 0));
+    }
+    done.sort_unstable();
+
+    // create a single tree from the buffer
+    done.into_iter().reduce(|acc, n| tcx.mk_const(Expr(Binop(Mul, n, acc)), n.ty()))
+}
+
 pub trait HasTyCtxt<'tcx>: HasDataLayout {
     fn tcx(&self) -> TyCtxt<'tcx>;
 }
@@ -636,7 +731,7 @@ where
                     variants: Variants::Single { index: variant_index },
                     fields: match NonZeroUsize::new(fields) {
                         Some(fields) => FieldsShape::Union(fields),
-                        None => FieldsShape::Arbitrary { offsets: vec![], memory_index: vec![] },
+                        None => FieldsShape::Arbitrary { offsets: IndexVec::new(), memory_index: IndexVec::new() },
                     },
                     abi: Abi::Uninhabited,
                     largest_niche: None,
@@ -730,7 +825,11 @@ where
                         */
                     };
 
-                    let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type() {
+                    let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type()
+                        // Projection eagerly bails out when the pointee references errors,
+                        // fall back to structurally deducing metadata.
+                        && !pointee.references_error()
+                    {
                         let metadata = tcx.normalize_erasing_regions(
                             cx.param_env(),
                             tcx.mk_projection(metadata_def_id, [pointee]),
@@ -794,7 +893,8 @@ where
                 ty::Adt(def, substs) => {
                     match this.variants {
                         Variants::Single { index } => {
-                            TyMaybeWithLayout::Ty(def.variant(index).fields[i].ty(tcx, substs))
+                            let field = &def.variant(index).fields[FieldIdx::from_usize(i)];
+                            TyMaybeWithLayout::Ty(field.ty(tcx, substs))
                         }
 
                         // Discriminant field for enums (where applicable).
@@ -1126,10 +1226,11 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: SpecAbi) ->
         | AvrNonBlockingInterrupt
         | CCmseNonSecureCall
         | Wasm
-        | RustIntrinsic
         | PlatformIntrinsic
         | Unadjusted => false,
-        Rust | RustCall | RustCold => tcx.sess.panic_strategy() == PanicStrategy::Unwind,
+        Rust | RustCall | RustCold | RustIntrinsic => {
+            tcx.sess.panic_strategy() == PanicStrategy::Unwind
+        }
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index dce18a585..2e516f291 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -34,6 +34,7 @@ use rustc_attr as attr;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::intern::Interned;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_data_structures::steal::Steal;
 use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
@@ -44,10 +45,12 @@ use rustc_index::vec::IndexVec;
 use rustc_macros::HashStable;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_serialize::{Decodable, Encodable};
+use rustc_session::lint::LintBuffer;
+pub use rustc_session::lint::RegisteredTools;
 use rustc_span::hygiene::MacroKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{ExpnId, ExpnKind, Span};
-use rustc_target::abi::{Align, Integer, IntegerType, VariantIdx};
+use rustc_target::abi::{Align, FieldIdx, Integer, IntegerType, VariantIdx};
 pub use rustc_target::abi::{ReprFlags, ReprOptions};
 use rustc_type_ir::WithCachedTypeInfo;
 pub use subst::*;
@@ -148,8 +151,6 @@ mod typeck_results;
 
 // Data types
 
-pub type RegisteredTools = FxHashSet<Ident>;
-
 pub struct ResolverOutputs {
     pub global_ctxt: ResolverGlobalCtxt,
     pub ast_lowering: ResolverAstLowering,
@@ -165,7 +166,8 @@ pub struct ResolverGlobalCtxt {
     pub effective_visibilities: EffectiveVisibilities,
     pub extern_crate_map: FxHashMap<LocalDefId, CrateNum>,
     pub maybe_unused_trait_imports: FxIndexSet<LocalDefId>,
-    pub reexport_map: FxHashMap<LocalDefId, Vec<ModChild>>,
+    pub module_children_non_reexports: LocalDefIdMap<Vec<LocalDefId>>,
+    pub module_children_reexports: LocalDefIdMap<Vec<ModChild>>,
     pub glob_map: FxHashMap<LocalDefId, FxHashSet<Symbol>>,
     pub main_def: Option<MainDefinition>,
     pub trait_impls: FxIndexMap<DefId, Vec<LocalDefId>>,
@@ -175,7 +177,6 @@ pub struct ResolverGlobalCtxt {
     /// Mapping from ident span to path span for paths that don't exist as written, but that
     /// exist under `std`. For example, wrote `str::from_utf8` instead of `std::str::from_utf8`.
     pub confused_type_with_std_module: FxHashMap<Span, Span>,
-    pub registered_tools: RegisteredTools,
     pub doc_link_resolutions: FxHashMap<LocalDefId, DocLinkResMap>,
     pub doc_link_traits_in_scope: FxHashMap<LocalDefId, Vec<DefId>>,
     pub all_macro_rules: FxHashMap<Symbol, Res<ast::NodeId>>,
@@ -209,6 +210,9 @@ pub struct ResolverAstLowering {
     pub builtin_macro_kinds: FxHashMap<LocalDefId, MacroKind>,
     /// List functions and methods for which lifetime elision was successful.
     pub lifetime_elision_allowed: FxHashSet<ast::NodeId>,
+
+    /// Lints that were emitted by the resolver and early lints.
+    pub lint_buffer: Steal<LintBuffer>,
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -325,12 +329,15 @@ pub struct ClosureSizeProfileData<'tcx> {
     pub after_feature_tys: Ty<'tcx>,
 }
 
-pub trait DefIdTree: Copy {
-    fn opt_parent(self, id: DefId) -> Option<DefId>;
+impl TyCtxt<'_> {
+    #[inline]
+    pub fn opt_parent(self, id: DefId) -> Option<DefId> {
+        self.def_key(id).parent.map(|index| DefId { index, ..id })
+    }
 
     #[inline]
     #[track_caller]
-    fn parent(self, id: DefId) -> DefId {
+    pub fn parent(self, id: DefId) -> DefId {
         match self.opt_parent(id) {
             Some(id) => id,
             // not `unwrap_or_else` to avoid breaking caller tracking
@@ -340,17 +347,17 @@ pub trait DefIdTree: Copy {
 
     #[inline]
     #[track_caller]
-    fn opt_local_parent(self, id: LocalDefId) -> Option<LocalDefId> {
+    pub fn opt_local_parent(self, id: LocalDefId) -> Option<LocalDefId> {
         self.opt_parent(id.to_def_id()).map(DefId::expect_local)
     }
 
     #[inline]
     #[track_caller]
-    fn local_parent(self, id: LocalDefId) -> LocalDefId {
+    pub fn local_parent(self, id: LocalDefId) -> LocalDefId {
         self.parent(id.to_def_id()).expect_local()
     }
 
-    fn is_descendant_of(self, mut descendant: DefId, ancestor: DefId) -> bool {
+    pub fn is_descendant_of(self, mut descendant: DefId, ancestor: DefId) -> bool {
         if descendant.krate != ancestor.krate {
             return false;
         }
@@ -365,13 +372,6 @@ pub trait DefIdTree: Copy {
     }
 }
 
-impl<'tcx> DefIdTree for TyCtxt<'tcx> {
-    #[inline]
-    fn opt_parent(self, id: DefId) -> Option<DefId> {
-        self.def_key(id).parent.map(|index| DefId { index, ..id })
-    }
-}
-
 impl<Id> Visibility<Id> {
     pub fn is_public(self) -> bool {
         matches!(self, Visibility::Public)
@@ -391,19 +391,19 @@ impl<Id: Into<DefId>> Visibility<Id> {
     }
 
     /// Returns `true` if an item with this visibility is accessible from the given module.
-    pub fn is_accessible_from(self, module: impl Into<DefId>, tree: impl DefIdTree) -> bool {
+    pub fn is_accessible_from(self, module: impl Into<DefId>, tcx: TyCtxt<'_>) -> bool {
         match self {
             // Public items are visible everywhere.
             Visibility::Public => true,
-            Visibility::Restricted(id) => tree.is_descendant_of(module.into(), id.into()),
+            Visibility::Restricted(id) => tcx.is_descendant_of(module.into(), id.into()),
         }
     }
 
     /// Returns `true` if this visibility is at least as accessible as the given visibility
-    pub fn is_at_least(self, vis: Visibility<impl Into<DefId>>, tree: impl DefIdTree) -> bool {
+    pub fn is_at_least(self, vis: Visibility<impl Into<DefId>>, tcx: TyCtxt<'_>) -> bool {
         match vis {
             Visibility::Public => self.is_public(),
-            Visibility::Restricted(id) => self.is_accessible_from(id, tree),
+            Visibility::Restricted(id) => self.is_accessible_from(id, tcx),
         }
     }
 }
@@ -544,7 +544,7 @@ impl<'tcx> Predicate<'tcx> {
             | PredicateKind::Clause(Clause::TypeOutlives(_))
             | PredicateKind::Clause(Clause::Projection(_))
             | PredicateKind::Clause(Clause::ConstArgHasType(..))
-            | PredicateKind::AliasEq(..)
+            | PredicateKind::AliasRelate(..)
             | PredicateKind::ObjectSafe(_)
             | PredicateKind::ClosureKind(_, _, _)
             | PredicateKind::Subtype(_)
@@ -641,7 +641,23 @@ pub enum PredicateKind<'tcx> {
     /// This predicate requires two terms to be equal to eachother.
     ///
     /// Only used for new solver
-    AliasEq(Term<'tcx>, Term<'tcx>),
+    AliasRelate(Term<'tcx>, Term<'tcx>, AliasRelationDirection),
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
+#[derive(HashStable, Debug)]
+pub enum AliasRelationDirection {
+    Equate,
+    Subtype,
+}
+
+impl std::fmt::Display for AliasRelationDirection {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            AliasRelationDirection::Equate => write!(f, "=="),
+            AliasRelationDirection::Subtype => write!(f, "<:"),
+        }
+    }
 }
 
 /// The crate outlives map is computed during typeck and contains the
@@ -977,11 +993,11 @@ impl<'tcx> Term<'tcx> {
         }
     }
 
-    /// This function returns `None` for `AliasKind::Opaque`.
+    /// This function returns the inner `AliasTy` if this term is a projection.
     ///
     /// FIXME: rename `AliasTy` to `AliasTerm` and make sure we correctly
     /// deal with constants.
-    pub fn to_alias_term_no_opaque(&self, tcx: TyCtxt<'tcx>) -> Option<AliasTy<'tcx>> {
+    pub fn to_projection_term(&self, tcx: TyCtxt<'tcx>) -> Option<AliasTy<'tcx>> {
         match self.unpack() {
             TermKind::Ty(ty) => match ty.kind() {
                 ty::Alias(kind, alias_ty) => match kind {
@@ -999,7 +1015,7 @@ impl<'tcx> Term<'tcx> {
 
     pub fn is_infer(&self) -> bool {
         match self.unpack() {
-            TermKind::Ty(ty) => ty.is_ty_or_numeric_infer(),
+            TermKind::Ty(ty) => ty.is_ty_var(),
             TermKind::Const(ct) => ct.is_ct_infer(),
         }
     }
@@ -1036,6 +1052,21 @@ impl<'tcx> TermKind<'tcx> {
     }
 }
 
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum ParamTerm {
+    Ty(ParamTy),
+    Const(ParamConst),
+}
+
+impl ParamTerm {
+    pub fn index(self) -> usize {
+        match self {
+            ParamTerm::Ty(ty) => ty.index as usize,
+            ParamTerm::Const(ct) => ct.index as usize,
+        }
+    }
+}
+
 /// This kind of predicate has no *direct* correspondent in the
 /// syntax, but it roughly corresponds to the syntactic forms:
 ///
@@ -1129,6 +1160,13 @@ impl<'tcx, T> ToPredicate<'tcx, T> for T {
     }
 }
 
+impl<'tcx> ToPredicate<'tcx> for PredicateKind<'tcx> {
+    #[inline(always)]
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        ty::Binder::dummy(self).to_predicate(tcx)
+    }
+}
+
 impl<'tcx> ToPredicate<'tcx> for Binder<'tcx, PredicateKind<'tcx>> {
     #[inline(always)]
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
@@ -1143,6 +1181,13 @@ impl<'tcx> ToPredicate<'tcx> for Clause<'tcx> {
     }
 }
 
+impl<'tcx> ToPredicate<'tcx> for TraitRef<'tcx> {
+    #[inline(always)]
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        ty::Binder::dummy(self).to_predicate(tcx)
+    }
+}
+
 impl<'tcx> ToPredicate<'tcx> for Binder<'tcx, TraitRef<'tcx>> {
     #[inline(always)]
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
@@ -1193,7 +1238,7 @@ impl<'tcx> Predicate<'tcx> {
             PredicateKind::Clause(Clause::Trait(t)) => Some(predicate.rebind(t)),
             PredicateKind::Clause(Clause::Projection(..))
             | PredicateKind::Clause(Clause::ConstArgHasType(..))
-            | PredicateKind::AliasEq(..)
+            | PredicateKind::AliasRelate(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
             | PredicateKind::Clause(Clause::RegionOutlives(..))
@@ -1214,7 +1259,7 @@ impl<'tcx> Predicate<'tcx> {
             PredicateKind::Clause(Clause::Projection(t)) => Some(predicate.rebind(t)),
             PredicateKind::Clause(Clause::Trait(..))
             | PredicateKind::Clause(Clause::ConstArgHasType(..))
-            | PredicateKind::AliasEq(..)
+            | PredicateKind::AliasRelate(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
             | PredicateKind::Clause(Clause::RegionOutlives(..))
@@ -1236,7 +1281,7 @@ impl<'tcx> Predicate<'tcx> {
             PredicateKind::Clause(Clause::Trait(..))
             | PredicateKind::Clause(Clause::ConstArgHasType(..))
             | PredicateKind::Clause(Clause::Projection(..))
-            | PredicateKind::AliasEq(..)
+            | PredicateKind::AliasRelate(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
             | PredicateKind::Clause(Clause::RegionOutlives(..))
@@ -1386,7 +1431,7 @@ impl<'tcx> OpaqueHiddenType<'tcx> {
         // lifetimes with 'static and remapping only those used in the
         // `impl Trait` return type, resulting in the parameters
         // shifting.
-        let id_substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
+        let id_substs = InternalSubsts::identity_for_item(tcx, def_id);
         debug!(?id_substs);
 
         // This zip may have several times the same lifetime in `substs` paired with a different
@@ -1410,12 +1455,12 @@ impl<'tcx> OpaqueHiddenType<'tcx> {
 #[derive(HashStable, TyEncodable, TyDecodable)]
 pub struct Placeholder<T> {
     pub universe: UniverseIndex,
-    pub name: T,
+    pub bound: T,
 }
 
-pub type PlaceholderRegion = Placeholder<BoundRegionKind>;
+pub type PlaceholderRegion = Placeholder<BoundRegion>;
 
-pub type PlaceholderType = Placeholder<BoundTyKind>;
+pub type PlaceholderType = Placeholder<BoundTy>;
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable)]
 #[derive(TyEncodable, TyDecodable, PartialOrd, Ord)]
@@ -1847,7 +1892,7 @@ pub struct VariantDef {
     /// Discriminant of this variant.
     pub discr: VariantDiscr,
     /// Fields of this variant.
-    pub fields: Vec<FieldDef>,
+    pub fields: IndexVec<FieldIdx, FieldDef>,
     /// Flags of the variant (e.g. is field list non-exhaustive)?
     flags: VariantFlags,
 }
@@ -1874,7 +1919,7 @@ impl VariantDef {
         variant_did: Option<DefId>,
         ctor: Option<(CtorKind, DefId)>,
         discr: VariantDiscr,
-        fields: Vec<FieldDef>,
+        fields: IndexVec<FieldIdx, FieldDef>,
         adt_kind: AdtKind,
         parent_did: DefId,
         recovered: bool,
@@ -2028,7 +2073,6 @@ impl<'tcx> FieldDef {
     }
 }
 
-pub type Attributes<'tcx> = impl Iterator<Item = &'tcx ast::Attribute>;
 #[derive(Debug, PartialEq, Eq)]
 pub enum ImplOverlapKind {
     /// These impls are always allowed to overlap.
@@ -2071,7 +2115,9 @@ pub enum ImplOverlapKind {
     Issue33140,
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, TyEncodable, TyDecodable, HashStable)]
+/// Useful source information about where a desugared associated type for an
+/// RPITIT originated from.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encodable, Decodable, HashStable)]
 pub enum ImplTraitInTraitData {
     Trait { fn_def_id: DefId, opaque_def_id: DefId },
     Impl { fn_def_id: DefId },
@@ -2214,26 +2260,37 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    pub fn find_field_index(self, ident: Ident, variant: &VariantDef) -> Option<usize> {
-        variant
-            .fields
-            .iter()
-            .position(|field| self.hygienic_eq(ident, field.ident(self), variant.def_id))
+    /// If the def-id is an associated type that was desugared from a
+    /// return-position `impl Trait` from a trait, then provide the source info
+    /// about where that RPITIT came from.
+    pub fn opt_rpitit_info(self, def_id: DefId) -> Option<ImplTraitInTraitData> {
+        if let DefKind::AssocTy = self.def_kind(def_id) {
+            self.associated_item(def_id).opt_rpitit_info
+        } else {
+            None
+        }
+    }
+
+    pub fn find_field_index(self, ident: Ident, variant: &VariantDef) -> Option<FieldIdx> {
+        variant.fields.iter_enumerated().find_map(|(i, field)| {
+            self.hygienic_eq(ident, field.ident(self), variant.def_id).then_some(i)
+        })
     }
 
     /// Returns `true` if the impls are the same polarity and the trait either
     /// has no items or is annotated `#[marker]` and prevents item overrides.
+    #[instrument(level = "debug", skip(self), ret)]
     pub fn impls_are_allowed_to_overlap(
         self,
         def_id1: DefId,
         def_id2: DefId,
     ) -> Option<ImplOverlapKind> {
+        let impl_trait_ref1 = self.impl_trait_ref(def_id1);
+        let impl_trait_ref2 = self.impl_trait_ref(def_id2);
         // If either trait impl references an error, they're allowed to overlap,
         // as one of them essentially doesn't exist.
-        if self.impl_trait_ref(def_id1).map_or(false, |tr| tr.subst_identity().references_error())
-            || self
-                .impl_trait_ref(def_id2)
-                .map_or(false, |tr| tr.subst_identity().references_error())
+        if impl_trait_ref1.map_or(false, |tr| tr.subst_identity().references_error())
+            || impl_trait_ref2.map_or(false, |tr| tr.subst_identity().references_error())
         {
             return Some(ImplOverlapKind::Permitted { marker: false });
         }
@@ -2241,19 +2298,11 @@ impl<'tcx> TyCtxt<'tcx> {
         match (self.impl_polarity(def_id1), self.impl_polarity(def_id2)) {
             (ImplPolarity::Reservation, _) | (_, ImplPolarity::Reservation) => {
                 // `#[rustc_reservation_impl]` impls don't overlap with anything
-                debug!(
-                    "impls_are_allowed_to_overlap({:?}, {:?}) = Some(Permitted) (reservations)",
-                    def_id1, def_id2
-                );
                 return Some(ImplOverlapKind::Permitted { marker: false });
             }
             (ImplPolarity::Positive, ImplPolarity::Negative)
             | (ImplPolarity::Negative, ImplPolarity::Positive) => {
                 // `impl AutoTrait for Type` + `impl !AutoTrait for Type`
-                debug!(
-                    "impls_are_allowed_to_overlap({:?}, {:?}) - None (differing polarities)",
-                    def_id1, def_id2
-                );
                 return None;
             }
             (ImplPolarity::Positive, ImplPolarity::Positive)
@@ -2261,38 +2310,25 @@ impl<'tcx> TyCtxt<'tcx> {
         };
 
         let is_marker_overlap = {
-            let is_marker_impl = |def_id: DefId| -> bool {
-                let trait_ref = self.impl_trait_ref(def_id);
+            let is_marker_impl = |trait_ref: Option<EarlyBinder<TraitRef<'_>>>| -> bool {
                 trait_ref.map_or(false, |tr| self.trait_def(tr.skip_binder().def_id).is_marker)
             };
-            is_marker_impl(def_id1) && is_marker_impl(def_id2)
+            is_marker_impl(impl_trait_ref1) && is_marker_impl(impl_trait_ref2)
         };
 
         if is_marker_overlap {
-            debug!(
-                "impls_are_allowed_to_overlap({:?}, {:?}) = Some(Permitted) (marker overlap)",
-                def_id1, def_id2
-            );
             Some(ImplOverlapKind::Permitted { marker: true })
         } else {
             if let Some(self_ty1) = self.issue33140_self_ty(def_id1) {
                 if let Some(self_ty2) = self.issue33140_self_ty(def_id2) {
                     if self_ty1 == self_ty2 {
-                        debug!(
-                            "impls_are_allowed_to_overlap({:?}, {:?}) - issue #33140 HACK",
-                            def_id1, def_id2
-                        );
                         return Some(ImplOverlapKind::Issue33140);
                     } else {
-                        debug!(
-                            "impls_are_allowed_to_overlap({:?}, {:?}) - found {:?} != {:?}",
-                            def_id1, def_id2, self_ty1, self_ty2
-                        );
+                        debug!("found {self_ty1:?} != {self_ty2:?}");
                     }
                 }
             }
 
-            debug!("impls_are_allowed_to_overlap({:?}, {:?}) = None", def_id1, def_id2);
             None
         }
     }
@@ -2349,7 +2385,9 @@ impl<'tcx> TyCtxt<'tcx> {
             | ty::InstanceDef::Virtual(..)
             | ty::InstanceDef::ClosureOnceShim { .. }
             | ty::InstanceDef::DropGlue(..)
-            | ty::InstanceDef::CloneShim(..) => self.mir_shims(instance),
+            | ty::InstanceDef::CloneShim(..)
+            | ty::InstanceDef::ThreadLocalShim(..)
+            | ty::InstanceDef::FnPtrAddrShim(..) => self.mir_shims(instance),
         }
     }
 
@@ -2363,7 +2401,12 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     /// Gets all attributes with the given name.
-    pub fn get_attrs(self, did: DefId, attr: Symbol) -> ty::Attributes<'tcx> {
+    pub fn get_attrs(
+        self,
+        did: impl Into<DefId>,
+        attr: Symbol,
+    ) -> impl Iterator<Item = &'tcx ast::Attribute> {
+        let did: DefId = did.into();
         let filter_fn = move |a: &&ast::Attribute| a.has_name(attr);
         if let Some(did) = did.as_local() {
             self.hir().attrs(self.hir().local_def_id_to_hir_id(did)).iter().filter(filter_fn)
@@ -2374,8 +2417,9 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    pub fn get_attr(self, did: DefId, attr: Symbol) -> Option<&'tcx ast::Attribute> {
+    pub fn get_attr(self, did: impl Into<DefId>, attr: Symbol) -> Option<&'tcx ast::Attribute> {
         if cfg!(debug_assertions) && !rustc_feature::is_valid_for_get_attr(attr) {
+            let did: DefId = did.into();
             bug!("get_attr: unexpected called with DefId `{:?}`, attr `{:?}`", did, attr);
         } else {
             self.get_attrs(did, attr).next()
@@ -2383,7 +2427,8 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     /// Determines whether an item is annotated with an attribute.
-    pub fn has_attr(self, did: DefId, attr: Symbol) -> bool {
+    pub fn has_attr(self, did: impl Into<DefId>, attr: Symbol) -> bool {
+        let did: DefId = did.into();
         if cfg!(debug_assertions) && !did.is_local() && rustc_feature::is_builtin_only_local(attr) {
             bug!("tried to access the `only_local` attribute `{}` from an extern crate", attr);
         } else {
@@ -2493,7 +2538,7 @@ impl<'tcx> TyCtxt<'tcx> {
         ident
     }
 
-    // FIXME(vincenzoapalzzo): move the HirId to a LocalDefId
+    // FIXME(vincenzopalazzo): move the HirId to a LocalDefId
     pub fn adjust_ident_and_get_scope(
         self,
         mut ident: Ident,
@@ -2540,12 +2585,18 @@ impl<'tcx> TyCtxt<'tcx> {
         matches!(self.trait_of_item(def_id), Some(trait_id) if self.has_attr(trait_id, sym::const_trait))
     }
 
-    pub fn impl_trait_in_trait_parent(self, mut def_id: DefId) -> DefId {
-        while let def_kind = self.def_kind(def_id) && def_kind != DefKind::AssocFn {
-            debug_assert_eq!(def_kind, DefKind::ImplTraitPlaceholder);
-            def_id = self.parent(def_id);
+    pub fn impl_trait_in_trait_parent_fn(self, mut def_id: DefId) -> DefId {
+        match self.opt_rpitit_info(def_id) {
+            Some(ImplTraitInTraitData::Trait { fn_def_id, .. })
+            | Some(ImplTraitInTraitData::Impl { fn_def_id, .. }) => fn_def_id,
+            None => {
+                while let def_kind = self.def_kind(def_id) && def_kind != DefKind::AssocFn {
+                    debug_assert_eq!(def_kind, DefKind::ImplTraitPlaceholder);
+                    def_id = self.parent(def_id);
+                }
+                def_id
+            }
         }
-        def_id
     }
 
     pub fn impl_method_has_trait_impl_trait_tys(self, def_id: DefId) -> bool {
@@ -2560,6 +2611,12 @@ impl<'tcx> TyCtxt<'tcx> {
 
         let Some(trait_item_def_id) = item.trait_item_def_id else { return false; };
 
+        if self.lower_impl_trait_in_trait_to_assoc_ty() {
+            return !self
+                .associated_types_for_impl_traits_in_associated_fn(trait_item_def_id)
+                .is_empty();
+        }
+
         // FIXME(RPITIT): This does a somewhat manual walk through the signature
         // of the trait fn to look for any RPITITs, but that's kinda doing a lot
         // of work. We can probably remove this when we refactor RPITITs to be
diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs
index 8849e7eab..7534d06ae 100644
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@@ -63,6 +63,7 @@ trivially_parameterized_over_tcx! {
     ty::DeducedParamAttrs,
     ty::Generics,
     ty::ImplPolarity,
+    ty::ImplTraitInTraitData,
     ty::ReprOptions,
     ty::TraitDef,
     ty::UnusedGenericParams,
diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs
index 021c20b58..d947d9604 100644
--- a/compiler/rustc_middle/src/ty/print/mod.rs
+++ b/compiler/rustc_middle/src/ty/print/mod.rs
@@ -1,5 +1,5 @@
 use crate::ty::GenericArg;
-use crate::ty::{self, DefIdTree, Ty, TyCtxt};
+use crate::ty::{self, Ty, TyCtxt};
 
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::sso::SsoHashSet;
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 6a053c368..72caadaf6 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -1,6 +1,6 @@
 use crate::mir::interpret::{AllocRange, GlobalAlloc, Pointer, Provenance, Scalar};
 use crate::ty::{
-    self, ConstInt, DefIdTree, ParamConst, ScalarInt, Term, TermKind, Ty, TyCtxt, TypeFoldable,
+    self, ConstInt, ParamConst, ScalarInt, Term, TermKind, Ty, TyCtxt, TypeFoldable,
     TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
 };
 use crate::ty::{GenericArg, GenericArgKind};
@@ -701,10 +701,12 @@ pub trait PrettyPrinter<'tcx>:
             ty::Error(_) => p!("[type error]"),
             ty::Param(ref param_ty) => p!(print(param_ty)),
             ty::Bound(debruijn, bound_ty) => match bound_ty.kind {
-                ty::BoundTyKind::Anon(bv) => {
-                    self.pretty_print_bound_var(debruijn, ty::BoundVar::from_u32(bv))?
-                }
-                ty::BoundTyKind::Param(_, s) => p!(write("{}", s)),
+                ty::BoundTyKind::Anon => self.pretty_print_bound_var(debruijn, bound_ty.var)?,
+                ty::BoundTyKind::Param(_, s) => match self.should_print_verbose() {
+                    true if debruijn == ty::INNERMOST => p!(write("^{}", s)),
+                    true => p!(write("^{}_{}", debruijn.index(), s)),
+                    false => p!(write("{}", s)),
+                },
             },
             ty::Adt(def, substs) => {
                 p!(print_def_path(def.did(), substs));
@@ -728,15 +730,15 @@ pub trait PrettyPrinter<'tcx>:
             }
             ty::Alias(ty::Projection, ref data) => {
                 if !(self.should_print_verbose() || NO_QUERIES.with(|q| q.get()))
-                    && self.tcx().def_kind(data.def_id) == DefKind::ImplTraitPlaceholder
+                    && self.tcx().is_impl_trait_in_trait(data.def_id)
                 {
                     return self.pretty_print_opaque_impl_type(data.def_id, data.substs);
                 } else {
                     p!(print(data))
                 }
             }
-            ty::Placeholder(placeholder) => match placeholder.name {
-                ty::BoundTyKind::Anon(_) => p!(write("Placeholder({:?})", placeholder)),
+            ty::Placeholder(placeholder) => match placeholder.bound.kind {
+                ty::BoundTyKind::Anon => p!(write("Placeholder({:?})", placeholder)),
                 ty::BoundTyKind::Param(_, name) => p!(write("{}", name)),
             },
             ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) => {
@@ -1345,7 +1347,7 @@ pub trait PrettyPrinter<'tcx>:
                             p!(write("{}::{}", self.tcx().crate_name(def.did.krate), self.tcx().def_path(def.did).to_string_no_crate_verbose()))
                         }
                     }
-                    defkind => bug!("`{:?}` has unexpcted defkind {:?}", ct, defkind),
+                    defkind => bug!("`{:?}` has unexpected defkind {:?}", ct, defkind),
                 }
             }
             ty::ConstKind::Infer(infer_ct) => {
@@ -2100,7 +2102,9 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
 
             ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
             | ty::ReFree(ty::FreeRegion { bound_region: br, .. })
-            | ty::RePlaceholder(ty::Placeholder { name: br, .. }) => {
+            | ty::RePlaceholder(ty::Placeholder {
+                bound: ty::BoundRegion { kind: br, .. }, ..
+            }) => {
                 if br.is_named() {
                     return true;
                 }
@@ -2177,7 +2181,9 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             }
             ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
             | ty::ReFree(ty::FreeRegion { bound_region: br, .. })
-            | ty::RePlaceholder(ty::Placeholder { name: br, .. }) => {
+            | ty::RePlaceholder(ty::Placeholder {
+                bound: ty::BoundRegion { kind: br, .. }, ..
+            }) => {
                 if let ty::BrNamed(_, name) = br && br.is_named() {
                     p!(write("{}", name));
                     return Ok(self);
@@ -2255,7 +2261,10 @@ impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> {
             ty::ReLateBound(db, br) if db >= self.current_index => {
                 *self.region_map.entry(br).or_insert_with(|| name(Some(db), self.current_index, br))
             }
-            ty::RePlaceholder(ty::PlaceholderRegion { name: kind, .. }) => {
+            ty::RePlaceholder(ty::PlaceholderRegion {
+                bound: ty::BoundRegion { kind, .. },
+                ..
+            }) => {
                 // If this is an anonymous placeholder, don't rename. Otherwise, in some
                 // async fns, we get a `for<'r> Send` bound
                 match kind {
@@ -2847,7 +2856,7 @@ define_print_and_forward_display! {
                 p!("the type `", print(ty), "` is found in the environment")
             }
             ty::PredicateKind::Ambiguous => p!("ambiguous"),
-            ty::PredicateKind::AliasEq(t1, t2) => p!(print(t1), " == ", print(t2)),
+            ty::PredicateKind::AliasRelate(t1, t2, dir) => p!(print(t1), write(" {} ", dir), print(t2)),
         }
     }
 
diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs
index 2bc51baf8..fa9fea723 100644
--- a/compiler/rustc_middle/src/ty/query.rs
+++ b/compiler/rustc_middle/src/ty/query.rs
@@ -17,7 +17,8 @@ use crate::mir::interpret::{
 };
 use crate::mir::interpret::{LitToConstError, LitToConstInput};
 use crate::mir::mono::CodegenUnit;
-use crate::query::Key;
+use crate::query::erase::{erase, restore, Erase};
+use crate::query::{AsLocalKey, Key};
 use crate::thir;
 use crate::traits::query::{
     CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
@@ -41,7 +42,7 @@ use rustc_arena::TypedArena;
 use rustc_ast as ast;
 use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_attr as attr;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
+use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::Lrc;
@@ -50,11 +51,15 @@ use rustc_data_structures::unord::UnordSet;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, DocLinkResMap};
-use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId};
+use rustc_hir::def_id::{
+    CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId, LocalDefIdMap, LocalDefIdSet,
+};
 use rustc_hir::hir_id::OwnerId;
 use rustc_hir::lang_items::{LangItem, LanguageItems};
 use rustc_hir::{Crate, ItemLocalId, TraitCandidate};
 use rustc_index::vec::IndexVec;
+pub(crate) use rustc_query_system::query::QueryJobId;
+use rustc_query_system::query::*;
 use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
 use rustc_session::cstore::{CrateDepKind, CrateSource};
 use rustc_session::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLib};
@@ -64,18 +69,19 @@ use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi;
 use rustc_target::spec::PanicStrategy;
+
+use std::marker::PhantomData;
 use std::mem;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::sync::Arc;
 
-pub(crate) use rustc_query_system::query::QueryJobId;
-use rustc_query_system::query::*;
-
 #[derive(Default)]
 pub struct QuerySystem<'tcx> {
     pub arenas: QueryArenas<'tcx>,
     pub caches: QueryCaches<'tcx>,
+    // Since we erase query value types we tell the typesystem about them with `PhantomData`.
+    _phantom_values: QueryPhantomValues<'tcx>,
 }
 
 #[derive(Copy, Clone)]
@@ -97,6 +103,11 @@ pub struct TyCtxtEnsure<'tcx> {
     pub tcx: TyCtxt<'tcx>,
 }
 
+#[derive(Copy, Clone)]
+pub struct TyCtxtEnsureWithValue<'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+}
+
 impl<'tcx> TyCtxt<'tcx> {
     /// Returns a transparent wrapper for `TyCtxt`, which ensures queries
     /// are executed instead of just returning their results.
@@ -105,6 +116,15 @@ impl<'tcx> TyCtxt<'tcx> {
         TyCtxtEnsure { tcx: self }
     }
 
+    /// Returns a transparent wrapper for `TyCtxt`, which ensures queries
+    /// are executed instead of just returning their results.
+    ///
+    /// This version verifies that the computed result exists in the cache before returning.
+    #[inline(always)]
+    pub fn ensure_with_value(self) -> TyCtxtEnsureWithValue<'tcx> {
+        TyCtxtEnsureWithValue { tcx: self }
+    }
+
     /// Returns a transparent wrapper for `TyCtxt` which uses
     /// `span` as the location of queries performed through it.
     #[inline(always)]
@@ -135,6 +155,20 @@ macro_rules! query_if_arena {
     };
 }
 
+/// If `separate_provide_if_extern`, then the key can be projected to its
+/// local key via `<$K as AsLocalKey>::LocalKey`.
+macro_rules! local_key_if_separate_extern {
+    ([] $($K:tt)*) => {
+        $($K)*
+    };
+    ([(separate_provide_extern) $($rest:tt)*] $($K:tt)*) => {
+        <$($K)* as AsLocalKey>::LocalKey
+    };
+    ([$other:tt $($modifiers:tt)*] $($K:tt)*) => {
+        local_key_if_separate_extern!([$($modifiers)*] $($K)*)
+    };
+}
+
 macro_rules! separate_provide_extern_decl {
     ([][$name:ident]) => {
         ()
@@ -196,6 +230,12 @@ macro_rules! define_callbacks {
             $(pub type $name<'tcx> = $($K)*;)*
         }
         #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_keys_local {
+            use super::*;
+
+            $(pub type $name<'tcx> = local_key_if_separate_extern!([$($modifiers)*] $($K)*);)*
+        }
+        #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_values {
             use super::*;
 
@@ -227,8 +267,8 @@ macro_rules! define_callbacks {
                 pub fn $name<'tcx>(
                     _tcx: TyCtxt<'tcx>,
                     value: query_provided::$name<'tcx>,
-                ) -> query_values::$name<'tcx> {
-                    query_if_arena!([$($modifiers)*]
+                ) -> Erase<query_values::$name<'tcx>> {
+                    erase(query_if_arena!([$($modifiers)*]
                         {
                             if mem::needs_drop::<query_provided::$name<'tcx>>() {
                                 &*_tcx.query_system.arenas.$name.alloc(value)
@@ -237,7 +277,7 @@ macro_rules! define_callbacks {
                             }
                         }
                         (value)
-                    )
+                    ))
                 }
             )*
         }
@@ -246,10 +286,40 @@ macro_rules! define_callbacks {
             use super::*;
 
             $(
-                pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
+                pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>>::Cache;
             )*
         }
 
+        $(
+            // Ensure that keys grow no larger than 64 bytes
+            #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+            const _: () = {
+                if mem::size_of::<query_keys::$name<'static>>() > 64 {
+                    panic!("{}", concat!(
+                        "the query `",
+                        stringify!($name),
+                        "` has a key type `",
+                        stringify!($($K)*),
+                        "` that is too large"
+                    ));
+                }
+            };
+
+            // Ensure that values grow no larger than 64 bytes
+            #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+            const _: () = {
+                if mem::size_of::<query_values::$name<'static>>() > 64 {
+                    panic!("{}", concat!(
+                        "the query `",
+                        stringify!($name),
+                        "` has a value type `",
+                        stringify!($V),
+                        "` that is too large"
+                    ));
+                }
+            };
+        )*
+
         pub struct QueryArenas<'tcx> {
             $($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
                 (WorkerLocal<TypedArena<<$V as Deref>::Target>>)
@@ -269,6 +339,11 @@ macro_rules! define_callbacks {
         }
 
         #[derive(Default)]
+        pub struct QueryPhantomValues<'tcx> {
+            $($(#[$attr])* pub $name: PhantomData<query_values::$name<'tcx>>,)*
+        }
+
+        #[derive(Default)]
         pub struct QueryCaches<'tcx> {
             $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)*
         }
@@ -282,7 +357,31 @@ macro_rules! define_callbacks {
 
                 match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
                     Some(_) => return,
-                    None => self.tcx.queries.$name(self.tcx, DUMMY_SP, key, QueryMode::Ensure),
+                    None => self.tcx.queries.$name(
+                        self.tcx,
+                        DUMMY_SP,
+                        key,
+                        QueryMode::Ensure { check_cache: false },
+                    ),
+                };
+            })*
+        }
+
+        impl<'tcx> TyCtxtEnsureWithValue<'tcx> {
+            $($(#[$attr])*
+            #[inline(always)]
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
+                let key = key.into_query_param();
+                opt_remap_env_constness!([$($modifiers)*][key]);
+
+                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
+                    Some(_) => return,
+                    None => self.tcx.queries.$name(
+                        self.tcx,
+                        DUMMY_SP,
+                        key,
+                        QueryMode::Ensure { check_cache: true },
+                    ),
                 };
             })*
         }
@@ -305,17 +404,17 @@ macro_rules! define_callbacks {
                 let key = key.into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
-                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
+                restore::<$V>(match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
                     Some(value) => value,
                     None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
-                }
+                })
             })*
         }
 
         pub struct Providers {
             $(pub $name: for<'tcx> fn(
                 TyCtxt<'tcx>,
-                query_keys::$name<'tcx>,
+                query_keys_local::$name<'tcx>,
             ) -> query_provided::$name<'tcx>,)*
         }
 
@@ -325,17 +424,14 @@ macro_rules! define_callbacks {
 
         impl Default for Providers {
             fn default() -> Self {
-                use crate::query::Key;
-
                 Providers {
                     $($name: |_, key| bug!(
-                        "`tcx.{}({:?})` is not supported for {} crate;\n\
+                        "`tcx.{}({:?})` is not supported for this key;\n\
                         hint: Queries can be either made to the local crate, or the external crate. \
                         This error means you tried to use it for one that's not supported.\n\
                         If that's not the case, {} was likely never assigned to a provider function.\n",
                         stringify!($name),
                         key,
-                        if key.query_crate_is_local() { "local" } else { "external" },
                         stringify!($name),
                     ),)*
                 }
@@ -372,7 +468,7 @@ macro_rules! define_callbacks {
                 span: Span,
                 key: query_keys::$name<'tcx>,
                 mode: QueryMode,
-            ) -> Option<$V>;)*
+            ) -> Option<Erase<$V>>;)*
         }
     };
 }
@@ -399,11 +495,13 @@ macro_rules! define_feedable {
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
                 let tcx = self.tcx;
-                let value = query_provided_to_value::$name(tcx, value);
+                let erased = query_provided_to_value::$name(tcx, value);
+                let value = restore::<$V>(erased);
                 let cache = &tcx.query_system.caches.$name;
 
                 match try_get_cached(tcx, cache, &key) {
                     Some(old) => {
+                        let old = restore::<$V>(old);
                         bug!(
                             "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}",
                             stringify!($name),
@@ -418,7 +516,7 @@ macro_rules! define_feedable {
                             &value,
                             hash_result!([$($modifiers)*]),
                         );
-                        cache.complete(key, value, dep_node_index);
+                        cache.complete(key, erased, dep_node_index);
                         value
                     }
                 }
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index 3fc5f5bed..46c931d61 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -22,8 +22,6 @@ pub enum Cause {
 pub trait TypeRelation<'tcx>: Sized {
     fn tcx(&self) -> TyCtxt<'tcx>;
 
-    fn intercrate(&self) -> bool;
-
     fn param_env(&self) -> ty::ParamEnv<'tcx>;
 
     /// Returns a static string we can use for printouts.
@@ -33,9 +31,6 @@ pub trait TypeRelation<'tcx>: Sized {
     /// relation. Just affects error messages.
     fn a_is_expected(&self) -> bool;
 
-    /// Used during coherence. If called, must emit an always-ambiguous obligation.
-    fn mark_ambiguous(&mut self);
-
     fn with_cause<F, R>(&mut self, _cause: Cause, f: F) -> R
     where
         F: FnOnce(&mut Self) -> R,
@@ -559,23 +554,16 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
             &ty::Alias(ty::Opaque, ty::AliasTy { def_id: a_def_id, substs: a_substs, .. }),
             &ty::Alias(ty::Opaque, ty::AliasTy { def_id: b_def_id, substs: b_substs, .. }),
         ) if a_def_id == b_def_id => {
-            if relation.intercrate() {
-                // During coherence, opaque types should be treated as equal to each other, even if their generic params
-                // differ, as they could resolve to the same hidden type, even for different generic params.
-                relation.mark_ambiguous();
-                Ok(a)
-            } else {
-                let opt_variances = tcx.variances_of(a_def_id);
-                let substs = relate_substs_with_variances(
-                    relation,
-                    a_def_id,
-                    opt_variances,
-                    a_substs,
-                    b_substs,
-                    false, // do not fetch `type_of(a_def_id)`, as it will cause a cycle
-                )?;
-                Ok(tcx.mk_opaque(a_def_id, substs))
-            }
+            let opt_variances = tcx.variances_of(a_def_id);
+            let substs = relate_substs_with_variances(
+                relation,
+                a_def_id,
+                opt_variances,
+                a_substs,
+                b_substs,
+                false, // do not fetch `type_of(a_def_id)`, as it will cause a cycle
+            )?;
+            Ok(tcx.mk_opaque(a_def_id, substs))
         }
 
         _ => Err(TypeError::Sorts(expected_found(relation, a, b))),
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index ef643531b..5c604bb6d 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -4,7 +4,7 @@
 //! to help with the tedium.
 
 use crate::mir::interpret;
-use crate::mir::{Field, ProjectionKind};
+use crate::mir::ProjectionKind;
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeSuperFoldable};
 use crate::ty::print::{with_no_trimmed_paths, FmtPrinter, Printer};
 use crate::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
@@ -68,7 +68,7 @@ impl<'tcx> fmt::Debug for ty::adjustment::Adjustment<'tcx> {
 impl fmt::Debug for ty::BoundRegionKind {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
-            ty::BrAnon(n, span) => write!(f, "BrAnon({n:?}, {span:?})"),
+            ty::BrAnon(span) => write!(f, "BrAnon({span:?})"),
             ty::BrNamed(did, name) => {
                 if did.is_crate_root() {
                     write!(f, "BrNamed({})", name)
@@ -177,7 +177,9 @@ impl<'tcx> fmt::Debug for ty::PredicateKind<'tcx> {
                 write!(f, "TypeWellFormedFromEnv({:?})", ty)
             }
             ty::PredicateKind::Ambiguous => write!(f, "Ambiguous"),
-            ty::PredicateKind::AliasEq(t1, t2) => write!(f, "AliasEq({t1:?}, {t2:?})"),
+            ty::PredicateKind::AliasRelate(t1, t2, dir) => {
+                write!(f, "AliasRelate({t1:?}, {dir:?}, {t2:?})")
+            }
         }
     }
 }
@@ -215,6 +217,7 @@ CloneLiftImpls! {
 // implementation and traversal implementations (the latter only for
 // TyCtxt<'_> interners).
 TrivialTypeTraversalAndLiftImpls! {
+    ::rustc_target::abi::FieldIdx,
     ::rustc_target::abi::VariantIdx,
     crate::middle::region::Scope,
     crate::ty::FloatTy,
@@ -250,8 +253,9 @@ TrivialTypeTraversalAndLiftImpls! {
     crate::ty::AssocItem,
     crate::ty::AssocKind,
     crate::ty::AliasKind,
-    crate::ty::Placeholder<crate::ty::BoundRegionKind>,
-    crate::ty::Placeholder<crate::ty::BoundTyKind>,
+    crate::ty::AliasRelationDirection,
+    crate::ty::Placeholder<crate::ty::BoundRegion>,
+    crate::ty::Placeholder<crate::ty::BoundTy>,
     crate::ty::ClosureKind,
     crate::ty::FreeRegion,
     crate::ty::InferTy,
@@ -265,7 +269,6 @@ TrivialTypeTraversalAndLiftImpls! {
     ::rustc_span::Span,
     ::rustc_span::symbol::Ident,
     ::rustc_errors::ErrorGuaranteed,
-    Field,
     interpret::Scalar,
     rustc_target::abi::Size,
     ty::BoundVar,
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index e6a73e8bb..96c1577d5 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -7,14 +7,15 @@ use crate::ty::subst::{GenericArg, InternalSubsts, SubstsRef};
 use crate::ty::visit::ValidateBoundVars;
 use crate::ty::InferTy::*;
 use crate::ty::{
-    self, AdtDef, DefIdTree, Discr, FallibleTypeFolder, Term, Ty, TyCtxt, TypeFlags, TypeFoldable,
-    TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
+    self, AdtDef, Discr, Term, Ty, TyCtxt, TypeFlags, TypeSuperVisitable, TypeVisitable,
+    TypeVisitableExt, TypeVisitor,
 };
 use crate::ty::{List, ParamEnv};
 use hir::def::DefKind;
 use polonius_engine::Atom;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::intern::Interned;
+use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_hir::LangItem;
@@ -22,8 +23,8 @@ use rustc_index::vec::Idx;
 use rustc_macros::HashStable;
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
-use rustc_target::abi::VariantIdx;
-use rustc_target::spec::abi;
+use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT};
+use rustc_target::spec::abi::{self, Abi};
 use std::borrow::Cow;
 use std::cmp::Ordering;
 use std::fmt;
@@ -60,7 +61,7 @@ pub struct FreeRegion {
 #[derive(HashStable)]
 pub enum BoundRegionKind {
     /// An anonymous region parameter for a given fn (&T)
-    BrAnon(u32, Option<Span>),
+    BrAnon(Option<Span>),
 
     /// Named region parameters for functions (a in &'a T)
     ///
@@ -107,15 +108,6 @@ impl BoundRegionKind {
             _ => None,
         }
     }
-
-    pub fn expect_anon(&self) -> u32 {
-        match *self {
-            BoundRegionKind::BrNamed(_, _) | BoundRegionKind::BrEnv => {
-                bug!("expected anon region: {self:?}")
-            }
-            BoundRegionKind::BrAnon(idx, _) => idx,
-        }
-    }
 }
 
 pub trait Article {
@@ -136,10 +128,6 @@ impl<'tcx> Article for TyKind<'tcx> {
     }
 }
 
-// `TyKind` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(TyKind<'_>, 32);
-
 /// A closure can be modeled as a struct that looks like:
 /// ```ignore (illustrative)
 /// struct Closure<'l0...'li, T0...Tj, CK, CS, U>(...U);
@@ -517,8 +505,7 @@ impl<'tcx> GeneratorSubsts<'tcx> {
     #[inline]
     pub fn variant_range(&self, def_id: DefId, tcx: TyCtxt<'tcx>) -> Range<VariantIdx> {
         // FIXME requires optimized MIR
-        let num_variants = tcx.generator_layout(def_id).unwrap().variant_fields.len();
-        VariantIdx::new(0)..VariantIdx::new(num_variants)
+        FIRST_VARIANT..tcx.generator_layout(def_id).unwrap().variant_fields.next_index()
     }
 
     /// The discriminant for the given variant. Panics if the `variant_index` is
@@ -878,8 +865,8 @@ impl<'tcx> PolyTraitRef<'tcx> {
     }
 }
 
-impl rustc_errors::IntoDiagnosticArg for PolyTraitRef<'_> {
-    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
+impl<'tcx> IntoDiagnosticArg for TraitRef<'tcx> {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
         self.to_string().into_diagnostic_arg()
     }
 }
@@ -924,6 +911,12 @@ impl<'tcx> ExistentialTraitRef<'tcx> {
     }
 }
 
+impl<'tcx> IntoDiagnosticArg for ExistentialTraitRef<'tcx> {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        self.to_string().into_diagnostic_arg()
+    }
+}
+
 pub type PolyExistentialTraitRef<'tcx> = Binder<'tcx, ExistentialTraitRef<'tcx>>;
 
 impl<'tcx> PolyExistentialTraitRef<'tcx> {
@@ -940,12 +933,6 @@ impl<'tcx> PolyExistentialTraitRef<'tcx> {
     }
 }
 
-impl rustc_errors::IntoDiagnosticArg for PolyExistentialTraitRef<'_> {
-    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
-        self.to_string().into_diagnostic_arg()
-    }
-}
-
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
 #[derive(HashStable)]
 pub enum BoundVariableKind {
@@ -1160,78 +1147,12 @@ impl<'tcx, T: IntoIterator> Binder<'tcx, T> {
     }
 }
 
-struct SkipBindersAt<'tcx> {
-    tcx: TyCtxt<'tcx>,
-    index: ty::DebruijnIndex,
-}
-
-impl<'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for SkipBindersAt<'tcx> {
-    type Error = ();
-
-    fn interner(&self) -> TyCtxt<'tcx> {
-        self.tcx
-    }
-
-    fn try_fold_binder<T>(&mut self, t: Binder<'tcx, T>) -> Result<Binder<'tcx, T>, Self::Error>
-    where
-        T: ty::TypeFoldable<TyCtxt<'tcx>>,
-    {
-        self.index.shift_in(1);
-        let value = t.try_map_bound(|t| t.try_fold_with(self));
-        self.index.shift_out(1);
-        value
-    }
-
-    fn try_fold_ty(&mut self, ty: Ty<'tcx>) -> Result<Ty<'tcx>, Self::Error> {
-        if !ty.has_escaping_bound_vars() {
-            Ok(ty)
-        } else if let ty::Bound(index, bv) = *ty.kind() {
-            if index == self.index {
-                Err(())
-            } else {
-                Ok(self.interner().mk_bound(index.shifted_out(1), bv))
-            }
-        } else {
-            ty.try_super_fold_with(self)
-        }
-    }
-
-    fn try_fold_region(&mut self, r: ty::Region<'tcx>) -> Result<ty::Region<'tcx>, Self::Error> {
-        if !r.has_escaping_bound_vars() {
-            Ok(r)
-        } else if let ty::ReLateBound(index, bv) = r.kind() {
-            if index == self.index {
-                Err(())
-            } else {
-                Ok(self.interner().mk_re_late_bound(index.shifted_out(1), bv))
-            }
-        } else {
-            r.try_super_fold_with(self)
-        }
-    }
-
-    fn try_fold_const(&mut self, ct: ty::Const<'tcx>) -> Result<ty::Const<'tcx>, Self::Error> {
-        if !ct.has_escaping_bound_vars() {
-            Ok(ct)
-        } else if let ty::ConstKind::Bound(index, bv) = ct.kind() {
-            if index == self.index {
-                Err(())
-            } else {
-                Ok(self.interner().mk_const(
-                    ty::ConstKind::Bound(index.shifted_out(1), bv),
-                    ct.ty().try_fold_with(self)?,
-                ))
-            }
-        } else {
-            ct.try_super_fold_with(self)
-        }
-    }
-
-    fn try_fold_predicate(
-        &mut self,
-        p: ty::Predicate<'tcx>,
-    ) -> Result<ty::Predicate<'tcx>, Self::Error> {
-        if !p.has_escaping_bound_vars() { Ok(p) } else { p.try_super_fold_with(self) }
+impl<'tcx, T> IntoDiagnosticArg for Binder<'tcx, T>
+where
+    T: IntoDiagnosticArg,
+{
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        self.0.into_diagnostic_arg()
     }
 }
 
@@ -1288,7 +1209,7 @@ impl<'tcx> AliasTy<'tcx> {
         match tcx.def_kind(self.def_id) {
             DefKind::AssocTy | DefKind::AssocConst => tcx.parent(self.def_id),
             DefKind::ImplTraitPlaceholder => {
-                tcx.parent(tcx.impl_trait_in_trait_parent(self.def_id))
+                tcx.parent(tcx.impl_trait_in_trait_parent_fn(self.def_id))
             }
             kind => bug!("expected a projection AliasTy; found {kind:?}"),
         }
@@ -1376,6 +1297,12 @@ impl<'tcx> FnSig<'tcx> {
     }
 }
 
+impl<'tcx> IntoDiagnosticArg for FnSig<'tcx> {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        self.to_string().into_diagnostic_arg()
+    }
+}
+
 pub type PolyFnSig<'tcx> = Binder<'tcx, FnSig<'tcx>>;
 
 impl<'tcx> PolyFnSig<'tcx> {
@@ -1403,6 +1330,18 @@ impl<'tcx> PolyFnSig<'tcx> {
     pub fn abi(&self) -> abi::Abi {
         self.skip_binder().abi
     }
+
+    pub fn is_fn_trait_compatible(&self) -> bool {
+        matches!(
+            self.skip_binder(),
+            ty::FnSig {
+                unsafety: rustc_hir::Unsafety::Normal,
+                abi: Abi::Rust,
+                c_variadic: false,
+                ..
+            }
+        )
+    }
 }
 
 pub type CanonicalPolyFnSig<'tcx> = Canonical<'tcx, Binder<'tcx, FnSig<'tcx>>>;
@@ -1522,22 +1461,13 @@ pub struct BoundTy {
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
 #[derive(HashStable)]
 pub enum BoundTyKind {
-    Anon(u32),
+    Anon,
     Param(DefId, Symbol),
 }
 
-impl BoundTyKind {
-    pub fn expect_anon(self) -> u32 {
-        match self {
-            BoundTyKind::Anon(i) => i,
-            _ => bug!(),
-        }
-    }
-}
-
 impl From<BoundVar> for BoundTy {
     fn from(var: BoundVar) -> Self {
-        BoundTy { var, kind: BoundTyKind::Anon(var.as_u32()) }
+        BoundTy { var, kind: BoundTyKind::Anon }
     }
 }
 
@@ -1616,19 +1546,24 @@ impl<'tcx> Region<'tcx> {
 
     pub fn get_name(self) -> Option<Symbol> {
         if self.has_name() {
-            let name = match *self {
+            match *self {
                 ty::ReEarlyBound(ebr) => Some(ebr.name),
                 ty::ReLateBound(_, br) => br.kind.get_name(),
                 ty::ReFree(fr) => fr.bound_region.get_name(),
                 ty::ReStatic => Some(kw::StaticLifetime),
-                ty::RePlaceholder(placeholder) => placeholder.name.get_name(),
+                ty::RePlaceholder(placeholder) => placeholder.bound.kind.get_name(),
                 _ => None,
-            };
-
-            return name;
+            }
+        } else {
+            None
         }
+    }
 
-        None
+    pub fn get_name_or_anon(self) -> Symbol {
+        match self.get_name() {
+            Some(name) => name,
+            None => sym::anon,
+        }
     }
 
     /// Is this region named by the user?
@@ -1639,7 +1574,7 @@ impl<'tcx> Region<'tcx> {
             ty::ReFree(fr) => fr.bound_region.is_named(),
             ty::ReStatic => true,
             ty::ReVar(..) => false,
-            ty::RePlaceholder(placeholder) => placeholder.name.is_named(),
+            ty::RePlaceholder(placeholder) => placeholder.bound.kind.is_named(),
             ty::ReErased => false,
             ty::ReError(_) => false,
         }
@@ -1762,10 +1697,10 @@ impl<'tcx> Region<'tcx> {
         matches!(self.kind(), ty::ReVar(_))
     }
 
-    pub fn as_var(self) -> Option<RegionVid> {
+    pub fn as_var(self) -> RegionVid {
         match self.kind() {
-            ty::ReVar(vid) => Some(vid),
-            _ => None,
+            ty::ReVar(vid) => vid,
+            _ => bug!("expected region {:?} to be of kind ReVar", self),
         }
     }
 }
@@ -1892,7 +1827,7 @@ impl<'tcx> Ty<'tcx> {
             Adt(def, substs) => {
                 assert!(def.repr().simd(), "`simd_size_and_type` called on non-SIMD type");
                 let variant = def.non_enum_variant();
-                let f0_ty = variant.fields[0].ty(tcx, substs);
+                let f0_ty = variant.fields[FieldIdx::from_u32(0)].ty(tcx, substs);
 
                 match f0_ty.kind() {
                     // If the first field is an array, we assume it is the only field and its
@@ -1902,7 +1837,7 @@ impl<'tcx> Ty<'tcx> {
                         // The way we evaluate the `N` in `[T; N]` here only works since we use
                         // `simd_size_and_type` post-monomorphization. It will probably start to ICE
                         // if we use it in generic code. See the `simd-array-trait` ui test.
-                        (f0_len.eval_target_usize(tcx, ParamEnv::empty()) as u64, *f0_elem_ty)
+                        (f0_len.eval_target_usize(tcx, ParamEnv::empty()), *f0_elem_ty)
                     }
                     // Otherwise, the fields of this Adt are the SIMD components (and we assume they
                     // all have the same type).
@@ -1914,11 +1849,6 @@ impl<'tcx> Ty<'tcx> {
     }
 
     #[inline]
-    pub fn is_region_ptr(self) -> bool {
-        matches!(self.kind(), Ref(..))
-    }
-
-    #[inline]
     pub fn is_mutable_ptr(self) -> bool {
         matches!(
             self.kind(),
@@ -1944,7 +1874,7 @@ impl<'tcx> Ty<'tcx> {
     /// Tests if this is any kind of primitive pointer type (reference, raw pointer, fn pointer).
     #[inline]
     pub fn is_any_ptr(self) -> bool {
-        self.is_region_ptr() || self.is_unsafe_ptr() || self.is_fn_ptr()
+        self.is_ref() || self.is_unsafe_ptr() || self.is_fn_ptr()
     }
 
     #[inline]
@@ -2508,3 +2438,14 @@ impl<'tcx> VarianceDiagInfo<'tcx> {
         }
     }
 }
+
+// Some types are used a lot. Make sure they don't unintentionally get bigger.
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+mod size_asserts {
+    use super::*;
+    use rustc_data_structures::static_assert_size;
+    // tidy-alphabetical-start
+    static_assert_size!(RegionKind<'_>, 28);
+    static_assert_size!(TyKind<'_>, 32);
+    // tidy-alphabetical-end
+}
diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs
index b090bd9d8..f05b87343 100644
--- a/compiler/rustc_middle/src/ty/subst.rs
+++ b/compiler/rustc_middle/src/ty/subst.rs
@@ -302,8 +302,8 @@ impl<'tcx> InternalSubsts<'tcx> {
     }
 
     /// Creates an `InternalSubsts` that maps each generic parameter to itself.
-    pub fn identity_for_item(tcx: TyCtxt<'tcx>, def_id: DefId) -> SubstsRef<'tcx> {
-        Self::for_item(tcx, def_id, |param, _| tcx.mk_param_from_def(param))
+    pub fn identity_for_item(tcx: TyCtxt<'tcx>, def_id: impl Into<DefId>) -> SubstsRef<'tcx> {
+        Self::for_item(tcx, def_id.into(), |param, _| tcx.mk_param_from_def(param))
     }
 
     /// Creates an `InternalSubsts` for generic parameter definitions,
diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs
index 233c0df2d..6747da7ab 100644
--- a/compiler/rustc_middle/src/ty/trait_def.rs
+++ b/compiler/rustc_middle/src/ty/trait_def.rs
@@ -1,5 +1,5 @@
 use crate::traits::specialization_graph;
-use crate::ty::fast_reject::{self, SimplifiedType, TreatParams};
+use crate::ty::fast_reject::{self, SimplifiedType, TreatParams, TreatProjections};
 use crate::ty::visit::TypeVisitableExt;
 use crate::ty::{Ident, Ty, TyCtxt};
 use hir::def_id::LOCAL_CRATE;
@@ -100,8 +100,9 @@ impl<'tcx> TraitDef {
 }
 
 impl<'tcx> TyCtxt<'tcx> {
-    pub fn for_each_impl<F: FnMut(DefId)>(self, def_id: DefId, mut f: F) {
-        let impls = self.trait_impls_of(def_id);
+    /// `trait_def_id` MUST BE the `DefId` of a trait.
+    pub fn for_each_impl<F: FnMut(DefId)>(self, trait_def_id: DefId, mut f: F) {
+        let impls = self.trait_impls_of(trait_def_id);
 
         for &impl_def_id in impls.blanket_impls.iter() {
             f(impl_def_id);
@@ -114,27 +115,45 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    /// Iterate over every impl that could possibly match the
-    /// self type `self_ty`.
-    pub fn for_each_relevant_impl<F: FnMut(DefId)>(
+    /// Iterate over every impl that could possibly match the self type `self_ty`.
+    ///
+    /// `trait_def_id` MUST BE the `DefId` of a trait.
+    pub fn for_each_relevant_impl(
         self,
-        def_id: DefId,
+        trait_def_id: DefId,
         self_ty: Ty<'tcx>,
-        mut f: F,
+        f: impl FnMut(DefId),
     ) {
-        let _: Option<()> = self.find_map_relevant_impl(def_id, self_ty, |did| {
-            f(did);
-            None
-        });
+        self.for_each_relevant_impl_treating_projections(
+            trait_def_id,
+            self_ty,
+            TreatProjections::ForLookup,
+            f,
+        )
     }
 
+    pub fn for_each_relevant_impl_treating_projections(
+        self,
+        trait_def_id: DefId,
+        self_ty: Ty<'tcx>,
+        treat_projections: TreatProjections,
+        mut f: impl FnMut(DefId),
+    ) {
+        let _: Option<()> =
+            self.find_map_relevant_impl(trait_def_id, self_ty, treat_projections, |did| {
+                f(did);
+                None
+            });
+    }
+
+    /// `trait_def_id` MUST BE the `DefId` of a trait.
     pub fn non_blanket_impls_for_ty(
         self,
-        def_id: DefId,
+        trait_def_id: DefId,
         self_ty: Ty<'tcx>,
     ) -> impl Iterator<Item = DefId> + 'tcx {
-        let impls = self.trait_impls_of(def_id);
-        if let Some(simp) = fast_reject::simplify_type(self, self_ty, TreatParams::AsInfer) {
+        let impls = self.trait_impls_of(trait_def_id);
+        if let Some(simp) = fast_reject::simplify_type(self, self_ty, TreatParams::AsCandidateKey) {
             if let Some(impls) = impls.non_blanket_impls.get(&simp) {
                 return impls.iter().copied();
             }
@@ -145,18 +164,21 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Applies function to every impl that could possibly match the self type `self_ty` and returns
     /// the first non-none value.
-    pub fn find_map_relevant_impl<T, F: FnMut(DefId) -> Option<T>>(
+    ///
+    /// `trait_def_id` MUST BE the `DefId` of a trait.
+    pub fn find_map_relevant_impl<T>(
         self,
-        def_id: DefId,
+        trait_def_id: DefId,
         self_ty: Ty<'tcx>,
-        mut f: F,
+        treat_projections: TreatProjections,
+        mut f: impl FnMut(DefId) -> Option<T>,
     ) -> Option<T> {
         // FIXME: This depends on the set of all impls for the trait. That is
         // unfortunate wrt. incremental compilation.
         //
         // If we want to be faster, we could have separate queries for
         // blanket and non-blanket impls, and compare them separately.
-        let impls = self.trait_impls_of(def_id);
+        let impls = self.trait_impls_of(trait_def_id);
 
         for &impl_def_id in impls.blanket_impls.iter() {
             if let result @ Some(_) = f(impl_def_id) {
@@ -164,14 +186,17 @@ impl<'tcx> TyCtxt<'tcx> {
             }
         }
 
-        // Note that we're using `TreatParams::AsPlaceholder` to query `non_blanket_impls` while using
-        // `TreatParams::AsInfer` while actually adding them.
-        //
+        // Note that we're using `TreatParams::ForLookup` to query `non_blanket_impls` while using
+        // `TreatParams::AsCandidateKey` while actually adding them.
+        let treat_params = match treat_projections {
+            TreatProjections::NextSolverLookup => TreatParams::NextSolverLookup,
+            TreatProjections::ForLookup => TreatParams::ForLookup,
+        };
         // This way, when searching for some impl for `T: Trait`, we do not look at any impls
         // whose outer level is not a parameter or projection. Especially for things like
         // `T: Clone` this is incredibly useful as we would otherwise look at all the impls
         // of `Clone` for `Option<T>`, `Vec<T>`, `ConcreteType` and so on.
-        if let Some(simp) = fast_reject::simplify_type(self, self_ty, TreatParams::AsPlaceholder) {
+        if let Some(simp) = fast_reject::simplify_type(self, self_ty, treat_params) {
             if let Some(impls) = impls.non_blanket_impls.get(&simp) {
                 for &impl_def_id in impls {
                     if let result @ Some(_) = f(impl_def_id) {
@@ -190,9 +215,11 @@ impl<'tcx> TyCtxt<'tcx> {
         None
     }
 
-    /// Returns an iterator containing all impls
-    pub fn all_impls(self, def_id: DefId) -> impl Iterator<Item = DefId> + 'tcx {
-        let TraitImpls { blanket_impls, non_blanket_impls } = self.trait_impls_of(def_id);
+    /// Returns an iterator containing all impls for `trait_def_id`.
+    ///
+    /// `trait_def_id` MUST BE the `DefId` of a trait.
+    pub fn all_impls(self, trait_def_id: DefId) -> impl Iterator<Item = DefId> + 'tcx {
+        let TraitImpls { blanket_impls, non_blanket_impls } = self.trait_impls_of(trait_def_id);
 
         blanket_impls.iter().chain(non_blanket_impls.iter().flat_map(|(_, v)| v)).cloned()
     }
@@ -231,7 +258,7 @@ pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> Trait
         }
 
         if let Some(simplified_self_ty) =
-            fast_reject::simplify_type(tcx, impl_self_ty, TreatParams::AsInfer)
+            fast_reject::simplify_type(tcx, impl_self_ty, TreatParams::AsCandidateKey)
         {
             impls.non_blanket_impls.entry(simplified_self_ty).or_default().push(impl_def_id);
         } else {
diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs
index 586958247..47943b94c 100644
--- a/compiler/rustc_middle/src/ty/typeck_results.rs
+++ b/compiler/rustc_middle/src/ty/typeck_results.rs
@@ -8,10 +8,9 @@ use crate::{
     },
 };
 use rustc_data_structures::{
-    fx::FxHashMap,
+    fx::{FxHashMap, FxIndexMap},
     sync::Lrc,
     unord::{UnordItems, UnordSet},
-    vec_map::VecMap,
 };
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
@@ -26,6 +25,7 @@ use rustc_macros::HashStable;
 use rustc_middle::mir::FakeReadCause;
 use rustc_session::Session;
 use rustc_span::Span;
+use rustc_target::abi::FieldIdx;
 use std::{collections::hash_map::Entry, hash::Hash, iter};
 
 use super::RvalueScopes;
@@ -43,7 +43,7 @@ pub struct TypeckResults<'tcx> {
     /// or patterns (`S { field }`). The index is often useful by itself, but to learn more
     /// about the field you also need definition of the variant to which the field
     /// belongs, but it may not exist if it's a tuple field (`tuple.0`).
-    field_indices: ItemLocalMap<usize>,
+    field_indices: ItemLocalMap<FieldIdx>,
 
     /// Stores the types for various nodes in the AST. Note that this table
     /// is not guaranteed to be populated outside inference. See
@@ -155,7 +155,7 @@ pub struct TypeckResults<'tcx> {
     /// by this function. We also store the
     /// type here, so that mir-borrowck can use it as a hint for figuring out hidden types,
     /// even if they are only set in dead code (which doesn't show up in MIR).
-    pub concrete_opaque_types: VecMap<LocalDefId, ty::OpaqueHiddenType<'tcx>>,
+    pub concrete_opaque_types: FxIndexMap<LocalDefId, ty::OpaqueHiddenType<'tcx>>,
 
     /// Tracks the minimum captures required for a closure;
     /// see `MinCaptureInformationMap` for more details.
@@ -314,19 +314,19 @@ impl<'tcx> TypeckResults<'tcx> {
         LocalTableInContextMut { hir_owner: self.hir_owner, data: &mut self.type_dependent_defs }
     }
 
-    pub fn field_indices(&self) -> LocalTableInContext<'_, usize> {
+    pub fn field_indices(&self) -> LocalTableInContext<'_, FieldIdx> {
         LocalTableInContext { hir_owner: self.hir_owner, data: &self.field_indices }
     }
 
-    pub fn field_indices_mut(&mut self) -> LocalTableInContextMut<'_, usize> {
+    pub fn field_indices_mut(&mut self) -> LocalTableInContextMut<'_, FieldIdx> {
         LocalTableInContextMut { hir_owner: self.hir_owner, data: &mut self.field_indices }
     }
 
-    pub fn field_index(&self, id: hir::HirId) -> usize {
+    pub fn field_index(&self, id: hir::HirId) -> FieldIdx {
         self.field_indices().get(id).cloned().expect("no index for a field")
     }
 
-    pub fn opt_field_index(&self, id: hir::HirId) -> Option<usize> {
+    pub fn opt_field_index(&self, id: hir::HirId) -> Option<FieldIdx> {
         self.field_indices().get(id).cloned()
     }
 
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 90270e0ee..c8a78ec03 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -2,10 +2,11 @@
 
 use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::mir;
+use crate::ty::fast_reject::TreatProjections;
 use crate::ty::layout::IntegerExt;
 use crate::ty::{
-    self, DefIdTree, FallibleTypeFolder, ToPredicate, Ty, TyCtxt, TypeFoldable, TypeFolder,
-    TypeSuperFoldable, TypeVisitableExt,
+    self, FallibleTypeFolder, ToPredicate, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
+    TypeVisitableExt,
 };
 use crate::ty::{GenericArgKind, SubstsRef};
 use rustc_apfloat::Float as _;
@@ -14,11 +15,12 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind, Res};
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_macros::HashStable;
-use rustc_span::{sym, DUMMY_SP};
+use rustc_session::Limit;
+use rustc_span::sym;
 use rustc_target::abi::{Integer, IntegerType, Size, TargetDataLayout};
 use rustc_target::spec::abi::Abi;
 use smallvec::SmallVec;
@@ -59,22 +61,13 @@ impl<'tcx> fmt::Display for Discr<'tcx> {
     }
 }
 
-fn int_size_and_signed<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (Size, bool) {
-    let (int, signed) = match *ty.kind() {
-        ty::Int(ity) => (Integer::from_int_ty(&tcx, ity), true),
-        ty::Uint(uty) => (Integer::from_uint_ty(&tcx, uty), false),
-        _ => bug!("non integer discriminant"),
-    };
-    (int.size(), signed)
-}
-
 impl<'tcx> Discr<'tcx> {
     /// Adds `1` to the value and wraps around if the maximum for the type is reached.
     pub fn wrap_incr(self, tcx: TyCtxt<'tcx>) -> Self {
         self.checked_add(tcx, 1).0
     }
     pub fn checked_add(self, tcx: TyCtxt<'tcx>, n: u128) -> (Self, bool) {
-        let (size, signed) = int_size_and_signed(tcx, self.ty);
+        let (size, signed) = self.ty.int_size_and_signed(tcx);
         let (val, oflo) = if signed {
             let min = size.signed_int_min();
             let max = size.signed_int_max();
@@ -233,17 +226,20 @@ impl<'tcx> TyCtxt<'tcx> {
         let recursion_limit = self.recursion_limit();
         for iteration in 0.. {
             if !recursion_limit.value_within_limit(iteration) {
-                return self.ty_error_with_message(
-                    DUMMY_SP,
-                    &format!("reached the recursion limit finding the struct tail for {}", ty),
-                );
+                let suggested_limit = match recursion_limit {
+                    Limit(0) => Limit(2),
+                    limit => limit * 2,
+                };
+                let reported =
+                    self.sess.emit_err(crate::error::RecursionLimitReached { ty, suggested_limit });
+                return self.ty_error(reported);
             }
             match *ty.kind() {
                 ty::Adt(def, substs) => {
                     if !def.is_struct() {
                         break;
                     }
-                    match def.non_enum_variant().fields.last() {
+                    match def.non_enum_variant().fields.raw.last() {
                         Some(field) => {
                             f();
                             ty = field.ty(self, substs);
@@ -317,7 +313,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 (&ty::Adt(a_def, a_substs), &ty::Adt(b_def, b_substs))
                     if a_def == b_def && a_def.is_struct() =>
                 {
-                    if let Some(f) = a_def.non_enum_variant().fields.last() {
+                    if let Some(f) = a_def.non_enum_variant().fields.raw.last() {
                         a = f.ty(self, a_substs);
                         b = f.ty(self, b_substs);
                     } else {
@@ -363,14 +359,20 @@ impl<'tcx> TyCtxt<'tcx> {
         self.ensure().coherent_trait(drop_trait);
 
         let ty = self.type_of(adt_did).subst_identity();
-        let (did, constness) = self.find_map_relevant_impl(drop_trait, ty, |impl_did| {
-            if let Some(item_id) = self.associated_item_def_ids(impl_did).first() {
-                if validate(self, impl_did).is_ok() {
-                    return Some((*item_id, self.constness(impl_did)));
+        let (did, constness) = self.find_map_relevant_impl(
+            drop_trait,
+            ty,
+            // FIXME: This could also be some other mode, like "unexpected"
+            TreatProjections::ForLookup,
+            |impl_did| {
+                if let Some(item_id) = self.associated_item_def_ids(impl_did).first() {
+                    if validate(self, impl_did).is_ok() {
+                        return Some((*item_id, self.constness(impl_did)));
+                    }
                 }
-            }
-            None
-        })?;
+                None
+            },
+        )?;
 
         Some(ty::Destructor { did, constness })
     }
@@ -599,6 +601,28 @@ impl<'tcx> TyCtxt<'tcx> {
         self.static_mutability(def_id) == Some(hir::Mutability::Mut)
     }
 
+    /// Returns `true` if the item pointed to by `def_id` is a thread local which needs a
+    /// thread local shim generated.
+    #[inline]
+    pub fn needs_thread_local_shim(self, def_id: DefId) -> bool {
+        !self.sess.target.dll_tls_export
+            && self.is_thread_local_static(def_id)
+            && !self.is_foreign_item(def_id)
+    }
+
+    /// Returns the type a reference to the thread local takes in MIR.
+    pub fn thread_local_ptr_ty(self, def_id: DefId) -> Ty<'tcx> {
+        let static_ty = self.type_of(def_id).subst_identity();
+        if self.is_mutable_static(def_id) {
+            self.mk_mut_ptr(static_ty)
+        } else if self.is_foreign_item(def_id) {
+            self.mk_imm_ptr(static_ty)
+        } else {
+            // FIXME: These things don't *really* have 'static lifetime.
+            self.mk_imm_ref(self.lifetimes.re_static, static_ty)
+        }
+    }
+
     /// Get the type of the pointer to the static that we use in MIR.
     pub fn static_ptr_ty(self, def_id: DefId) -> Ty<'tcx> {
         // Make sure that any constants in the static's type are evaluated.
@@ -684,10 +708,6 @@ impl<'tcx> TyCtxt<'tcx> {
         ty::EarlyBinder(self.explicit_item_bounds(def_id))
     }
 
-    pub fn bound_impl_subject(self, def_id: DefId) -> ty::EarlyBinder<ty::ImplSubject<'tcx>> {
-        ty::EarlyBinder(self.impl_subject(def_id))
-    }
-
     /// Returns names of captured upvars for closures and generators.
     ///
     /// Here are some examples:
@@ -922,12 +942,21 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for OpaqueTypeExpander<'tcx> {
 }
 
 impl<'tcx> Ty<'tcx> {
+    pub fn int_size_and_signed(self, tcx: TyCtxt<'tcx>) -> (Size, bool) {
+        let (int, signed) = match *self.kind() {
+            ty::Int(ity) => (Integer::from_int_ty(&tcx, ity), true),
+            ty::Uint(uty) => (Integer::from_uint_ty(&tcx, uty), false),
+            _ => bug!("non integer discriminant"),
+        };
+        (int.size(), signed)
+    }
+
     /// Returns the maximum value for the given numeric type (including `char`s)
     /// or returns `None` if the type is not numeric.
     pub fn numeric_max_val(self, tcx: TyCtxt<'tcx>) -> Option<ty::Const<'tcx>> {
         let val = match self.kind() {
             ty::Int(_) | ty::Uint(_) => {
-                let (size, signed) = int_size_and_signed(tcx, self);
+                let (size, signed) = self.int_size_and_signed(tcx);
                 let val =
                     if signed { size.signed_int_max() as u128 } else { size.unsigned_int_max() };
                 Some(val)
@@ -948,7 +977,7 @@ impl<'tcx> Ty<'tcx> {
     pub fn numeric_min_val(self, tcx: TyCtxt<'tcx>) -> Option<ty::Const<'tcx>> {
         let val = match self.kind() {
             ty::Int(_) | ty::Uint(_) => {
-                let (size, signed) = int_size_and_signed(tcx, self);
+                let (size, signed) = self.int_size_and_signed(tcx);
                 let val = if signed { size.truncate(size.signed_int_min() as u128) } else { 0 };
                 Some(val)
             }
@@ -1432,8 +1461,7 @@ pub fn reveal_opaque_types_in_bounds<'tcx>(
 }
 
 /// Determines whether an item is annotated with `doc(hidden)`.
-fn is_doc_hidden(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
-    assert!(def_id.is_local());
+fn is_doc_hidden(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     tcx.get_attrs(def_id, sym::doc)
         .filter_map(|attr| attr.meta_item_list())
         .any(|items| items.iter().any(|item| item.has_name(sym::hidden)))
@@ -1447,7 +1475,7 @@ pub fn is_doc_notable_trait(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
 }
 
 /// Determines whether an item is an intrinsic by Abi.
-pub fn is_intrinsic(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
+pub fn is_intrinsic(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     matches!(tcx.fn_sig(def_id).skip_binder().abi(), Abi::RustIntrinsic | Abi::PlatformIntrinsic)
 }
 
diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs
index 6814cadb9..08a62c900 100644
--- a/compiler/rustc_middle/src/ty/visit.rs
+++ b/compiler/rustc_middle/src/ty/visit.rs
@@ -83,6 +83,9 @@ pub trait TypeVisitableExt<'tcx>: TypeVisitable<TyCtxt<'tcx>> {
                 | TypeFlags::HAS_CT_PLACEHOLDER,
         )
     }
+    fn has_non_region_placeholders(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_TY_PLACEHOLDER | TypeFlags::HAS_CT_PLACEHOLDER)
+    }
     fn needs_subst(&self) -> bool {
         self.has_type_flags(TypeFlags::NEEDS_SUBST)
     }
diff --git a/compiler/rustc_middle/src/values.rs b/compiler/rustc_middle/src/values.rs
index c4f526dbd..55aa4fcff 100644
--- a/compiler/rustc_middle/src/values.rs
+++ b/compiler/rustc_middle/src/values.rs
@@ -4,7 +4,7 @@ use rustc_errors::{pluralize, struct_span_err, Applicability, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_middle::ty::Representability;
-use rustc_middle::ty::{self, DefIdTree, Ty, TyCtxt};
+use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_query_system::query::QueryInfo;
 use rustc_query_system::Value;
 use rustc_span::def_id::LocalDefId;
diff --git a/compiler/rustc_mir_build/locales/en-US.ftl b/compiler/rustc_mir_build/messages.ftl
index 93e7fb330..f346cd483 100644
--- a/compiler/rustc_mir_build/locales/en-US.ftl
+++ b/compiler/rustc_mir_build/messages.ftl
@@ -239,19 +239,9 @@ mir_build_trailing_irrefutable_let_patterns = trailing irrefutable {$count ->
     } into the body
 
 mir_build_bindings_with_variant_name =
-    pattern binding `{$ident}` is named the same as one of the variants of the type `{$ty_path}`
+    pattern binding `{$name}` is named the same as one of the variants of the type `{$ty_path}`
     .suggestion = to match on the variant, qualify the path
 
-mir_build_irrefutable_let_patterns_generic_let = irrefutable `let` {$count ->
-        [one] pattern
-        *[other] patterns
-    }
-    .note = {$count ->
-        [one] this pattern
-        *[other] these patterns
-    } will always match, so the `let` is useless
-    .help = consider removing `let`
-
 mir_build_irrefutable_let_patterns_if_let = irrefutable `if let` {$count ->
         [one] pattern
         *[other] patterns
@@ -331,6 +321,10 @@ mir_build_indirect_structural_match =
 mir_build_nontrivial_structural_match =
     to use a constant of type `{$non_sm_ty}` in a pattern, the constant's initializer must be trivial or `{$non_sm_ty}` must be annotated with `#[derive(PartialEq, Eq)]`
 
+mir_build_type_not_structural_tip = the traits must be derived, manual `impl`s are not sufficient
+
+mir_build_type_not_structural_more_info = see https://doc.rust-lang.org/stable/std/marker/trait.StructuralEq.html for details
+
 mir_build_overlapping_range_endpoints = multiple patterns overlap on their endpoints
     .range = ... with this range
     .note = you likely meant to write mutually exclusive ranges
@@ -353,15 +347,13 @@ mir_build_inform_irrefutable = `let` bindings require an "irrefutable pattern",
 
 mir_build_more_information = for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html
 
-mir_build_res_defined_here = {$res} defined here
-
 mir_build_adt_defined_here = `{$ty}` defined here
 
 mir_build_variant_defined_here = not covered
 
 mir_build_interpreted_as_const = introduce a variable instead
 
-mir_build_confused = missing patterns are not covered because `{$variable}` is interpreted as {$article} {$res} pattern, not a new variable
+mir_build_confused = missing patterns are not covered because `{$variable}` is interpreted as a constant pattern, not a new variable
 
 mir_build_suggest_if_let = you might want to use `if let` to ignore the {$count ->
         [one] variant that isn't
diff --git a/compiler/rustc_mir_build/src/build/block.rs b/compiler/rustc_mir_build/src/build/block.rs
index 2643d33ce..609ab1928 100644
--- a/compiler/rustc_mir_build/src/build/block.rs
+++ b/compiler/rustc_mir_build/src/build/block.rs
@@ -115,6 +115,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     initializer: Some(initializer),
                     lint_level,
                     else_block: Some(else_block),
+                    span: _,
                 } => {
                     // When lowering the statement `let <pat> = <expr> else { <else> };`,
                     // the `<else>` block is nested in the parent scope enclosing this statement.
@@ -278,6 +279,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     initializer,
                     lint_level,
                     else_block: None,
+                    span: _,
                 } => {
                     let ignores_expr_result = matches!(pattern.kind, PatKind::Wild);
                     this.block_context.push(BlockFrame::Statement { ignores_expr_result });
diff --git a/compiler/rustc_mir_build/src/build/cfg.rs b/compiler/rustc_mir_build/src/build/cfg.rs
index d7b4b1f73..4f1623b4c 100644
--- a/compiler/rustc_mir_build/src/build/cfg.rs
+++ b/compiler/rustc_mir_build/src/build/cfg.rs
@@ -90,6 +90,17 @@ impl<'tcx> CFG<'tcx> {
         self.push(block, stmt);
     }
 
+    pub(crate) fn push_place_mention(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        place: Place<'tcx>,
+    ) {
+        let kind = StatementKind::PlaceMention(Box::new(place));
+        let stmt = Statement { source_info, kind };
+        self.push(block, stmt);
+    }
+
     pub(crate) fn terminate(
         &mut self,
         block: BasicBlock,
diff --git a/compiler/rustc_mir_build/src/build/custom/mod.rs b/compiler/rustc_mir_build/src/build/custom/mod.rs
index 33fdc1901..d385153ba 100644
--- a/compiler/rustc_mir_build/src/build/custom/mod.rs
+++ b/compiler/rustc_mir_build/src/build/custom/mod.rs
@@ -21,7 +21,7 @@ use rustc_ast::Attribute;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::DefId;
 use rustc_hir::HirId;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::{
     mir::*,
     thir::*,
@@ -37,7 +37,7 @@ pub(super) fn build_custom_mir<'tcx>(
     hir_id: HirId,
     thir: &Thir<'tcx>,
     expr: ExprId,
-    params: &IndexVec<ParamId, Param<'tcx>>,
+    params: &IndexSlice<ParamId, Param<'tcx>>,
     return_ty: Ty<'tcx>,
     return_ty_span: Span,
     span: Span,
@@ -49,7 +49,7 @@ pub(super) fn build_custom_mir<'tcx>(
         phase: MirPhase::Built,
         source_scopes: IndexVec::new(),
         generator: None,
-        local_decls: LocalDecls::new(),
+        local_decls: IndexVec::new(),
         user_type_annotations: IndexVec::new(),
         arg_count: params.len(),
         spread_arg: None,
diff --git a/compiler/rustc_mir_build/src/build/custom/parse.rs b/compiler/rustc_mir_build/src/build/custom/parse.rs
index d72770e70..12b2f5d80 100644
--- a/compiler/rustc_mir_build/src/build/custom/parse.rs
+++ b/compiler/rustc_mir_build/src/build/custom/parse.rs
@@ -1,4 +1,4 @@
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::IndexSlice;
 use rustc_middle::{mir::*, thir::*, ty::Ty};
 use rustc_span::Span;
 
@@ -81,7 +81,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
         }
     }
 
-    pub fn parse_args(&mut self, params: &IndexVec<ParamId, Param<'tcx>>) -> PResult<()> {
+    pub fn parse_args(&mut self, params: &IndexSlice<ParamId, Param<'tcx>>) -> PResult<()> {
         for param in params.iter() {
             let (var, span) = {
                 let pat = param.pat.as_ref().unwrap();
diff --git a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
index dbba529ae..931fe1b24 100644
--- a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
+++ b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
@@ -1,8 +1,9 @@
 use rustc_middle::mir::interpret::{ConstValue, Scalar};
 use rustc_middle::mir::tcx::PlaceTy;
+use rustc_middle::ty::cast::mir_cast_kind;
 use rustc_middle::{mir::*, thir::*, ty};
 use rustc_span::Span;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 
 use crate::build::custom::ParseError;
 use crate::build::expr::as_constant::as_constant_inner;
@@ -55,15 +56,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
                 Ok(TerminatorKind::Drop {
                     place: self.parse_place(args[0])?,
                     target: self.parse_block(args[1])?,
-                    unwind: None,
-                })
-            },
-            @call("mir_drop_and_replace", args) => {
-                Ok(TerminatorKind::DropAndReplace {
-                    place: self.parse_place(args[0])?,
-                    value: self.parse_operand(args[1])?,
-                    target: self.parse_block(args[2])?,
-                    unwind: None,
+                    unwind: UnwindAction::Continue,
                 })
             },
             @call("mir_call", args) => {
@@ -133,7 +126,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
                     args,
                     destination,
                     target: Some(target),
-                    cleanup: None,
+                    unwind: UnwindAction::Continue,
                     from_hir_call: *from_hir_call,
                     fn_span: *fn_span,
                 })
@@ -142,8 +135,12 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
     }
 
     fn parse_rvalue(&self, expr_id: ExprId) -> PResult<Rvalue<'tcx>> {
-        parse_by_kind!(self, expr_id, _, "rvalue",
+        parse_by_kind!(self, expr_id, expr, "rvalue",
             @call("mir_discriminant", args) => self.parse_place(args[0]).map(Rvalue::Discriminant),
+            @call("mir_cast_transmute", args) => {
+                let source = self.parse_operand(args[0])?;
+                Ok(Rvalue::Cast(CastKind::Transmute, source, expr.ty))
+            },
             @call("mir_checked", args) => {
                 parse_by_kind!(self, args[0], _, "binary op",
                     ExprKind::Binary { op, lhs, rhs } => Ok(Rvalue::CheckedBinaryOp(
@@ -151,6 +148,11 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
                     )),
                 )
             },
+            @call("mir_offset", args) => {
+                let ptr = self.parse_operand(args[0])?;
+                let offset = self.parse_operand(args[1])?;
+                Ok(Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr, offset))))
+            },
             @call("mir_len", args) => Ok(Rvalue::Len(self.parse_place(args[0])?)),
             ExprKind::Borrow { borrow_kind, arg } => Ok(
                 Rvalue::Ref(self.tcx.lifetimes.re_erased, *borrow_kind, self.parse_place(*arg)?)
@@ -167,6 +169,34 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
             ExprKind::Repeat { value, count } => Ok(
                 Rvalue::Repeat(self.parse_operand(*value)?, *count)
             ),
+            ExprKind::Cast { source } => {
+                let source = self.parse_operand(*source)?;
+                let source_ty = source.ty(self.body.local_decls(), self.tcx);
+                let cast_kind = mir_cast_kind(source_ty, expr.ty);
+                Ok(Rvalue::Cast(cast_kind, source, expr.ty))
+            },
+            ExprKind::Tuple { fields } => Ok(
+                Rvalue::Aggregate(
+                    Box::new(AggregateKind::Tuple),
+                    fields.iter().map(|e| self.parse_operand(*e)).collect::<Result<_, _>>()?
+                )
+            ),
+            ExprKind::Array { fields } => {
+                let elem_ty = expr.ty.builtin_index().expect("ty must be an array");
+                Ok(Rvalue::Aggregate(
+                    Box::new(AggregateKind::Array(elem_ty)),
+                    fields.iter().map(|e| self.parse_operand(*e)).collect::<Result<_, _>>()?
+                ))
+            },
+            ExprKind::Adt(box AdtExpr{ adt_def, variant_index, substs, fields, .. }) => {
+                let is_union = adt_def.is_union();
+                let active_field_index = is_union.then(|| fields[0].name);
+
+                Ok(Rvalue::Aggregate(
+                    Box::new(AggregateKind::Adt(adt_def.did(), *variant_index, substs, None, active_field_index)),
+                    fields.iter().map(|f| self.parse_operand(f.expr)).collect::<Result<_, _>>()?
+                ))
+            },
             _ => self.parse_operand(expr_id).map(Rvalue::Use),
         )
     }
@@ -198,7 +228,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
         let (parent, proj) = parse_by_kind!(self, expr_id, expr, "place",
             @call("mir_field", args) => {
                 let (parent, ty) = self.parse_place_inner(args[0])?;
-                let field = Field::from_u32(self.parse_integer_literal(args[1])? as u32);
+                let field = FieldIdx::from_u32(self.parse_integer_literal(args[1])? as u32);
                 let field_ty = ty.field_ty(self.tcx, field);
                 let proj = PlaceElem::Field(field, field_ty);
                 let place = parent.project_deeper(&[proj], self.tcx);
diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
index cfacb5ea3..99291740a 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
@@ -62,21 +62,21 @@ pub fn as_constant_inner<'tcx>(
             Constant { span, user_ty: None, literal }
         }
         ExprKind::NonHirLiteral { lit, ref user_ty } => {
-            let user_ty = user_ty.as_ref().map(push_cuta).flatten();
+            let user_ty = user_ty.as_ref().and_then(push_cuta);
 
             let literal = ConstantKind::Val(ConstValue::Scalar(Scalar::Int(lit)), ty);
 
             Constant { span, user_ty, literal }
         }
         ExprKind::ZstLiteral { ref user_ty } => {
-            let user_ty = user_ty.as_ref().map(push_cuta).flatten();
+            let user_ty = user_ty.as_ref().and_then(push_cuta);
 
             let literal = ConstantKind::Val(ConstValue::ZeroSized, ty);
 
             Constant { span, user_ty, literal }
         }
         ExprKind::NamedConst { def_id, substs, ref user_ty } => {
-            let user_ty = user_ty.as_ref().map(push_cuta).flatten();
+            let user_ty = user_ty.as_ref().and_then(push_cuta);
 
             let uneval = mir::UnevaluatedConst::new(ty::WithOptConstParam::unknown(def_id), substs);
             let literal = ConstantKind::Unevaluated(uneval, ty);
diff --git a/compiler/rustc_mir_build/src/build/expr/as_operand.rs b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
index ff3198847..6941da331 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_operand.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
@@ -20,7 +20,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         expr: &Expr<'tcx>,
     ) -> BlockAnd<Operand<'tcx>> {
         let local_scope = self.local_scope();
-        self.as_operand(block, Some(local_scope), expr, None, NeedsTemporary::Maybe)
+        self.as_operand(block, Some(local_scope), expr, LocalInfo::Boring, NeedsTemporary::Maybe)
     }
 
     /// Returns an operand suitable for use until the end of the current scope expression and
@@ -102,7 +102,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         mut block: BasicBlock,
         scope: Option<region::Scope>,
         expr: &Expr<'tcx>,
-        local_info: Option<Box<LocalInfo<'tcx>>>,
+        local_info: LocalInfo<'tcx>,
         needs_temporary: NeedsTemporary,
     ) -> BlockAnd<Operand<'tcx>> {
         let this = self;
@@ -124,8 +124,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
             Category::Constant | Category::Place | Category::Rvalue(..) => {
                 let operand = unpack!(block = this.as_temp(block, scope, expr, Mutability::Mut));
-                if this.local_decls[operand].local_info.is_none() {
-                    this.local_decls[operand].local_info = local_info;
+                // Overwrite temp local info if we have something more interesting to record.
+                if !matches!(local_info, LocalInfo::Boring) {
+                    let decl_info = this.local_decls[operand].local_info.as_mut().assert_crate_local();
+                    if let LocalInfo::Boring | LocalInfo::BlockTailTemp(_) = **decl_info {
+                        **decl_info = local_info;
+                    }
                 }
                 block.and(Operand::Move(Place::from(operand)))
             }
@@ -178,6 +182,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
         }
 
-        this.as_operand(block, scope, expr, None, NeedsTemporary::Maybe)
+        this.as_operand(block, scope, expr, LocalInfo::Boring, NeedsTemporary::Maybe)
     }
 }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs
index 33200b80a..fb775766c 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs
@@ -13,9 +13,7 @@ use rustc_middle::thir::*;
 use rustc_middle::ty::AdtDef;
 use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, Variance};
 use rustc_span::Span;
-use rustc_target::abi::VariantIdx;
-
-use rustc_index::vec::Idx;
+use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT};
 
 use std::assert_matches::assert_matches;
 use std::iter;
@@ -91,8 +89,8 @@ fn convert_to_hir_projections_and_truncate_for_capture(
         let hir_projection = match mir_projection {
             ProjectionElem::Deref => HirProjectionKind::Deref,
             ProjectionElem::Field(field, _) => {
-                let variant = variant.unwrap_or(VariantIdx::new(0));
-                HirProjectionKind::Field(field.index() as u32, variant)
+                let variant = variant.unwrap_or(FIRST_VARIANT);
+                HirProjectionKind::Field(*field, variant)
             }
             ProjectionElem::Downcast(.., idx) => {
                 // We don't expect to see multi-variant enums here, as earlier
@@ -295,7 +293,7 @@ impl<'tcx> PlaceBuilder<'tcx> {
         &self.projection
     }
 
-    pub(crate) fn field(self, f: Field, ty: Ty<'tcx>) -> Self {
+    pub(crate) fn field(self, f: FieldIdx, ty: Ty<'tcx>) -> Self {
         self.project(PlaceElem::Field(f, ty))
     }
 
diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
index a4e48c154..8631749a5 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
@@ -1,14 +1,15 @@
 //! See docs in `build/expr/mod.rs`.
 
-use rustc_index::vec::Idx;
+use rustc_index::vec::{Idx, IndexVec};
 use rustc_middle::ty::util::IntTypeExt;
-use rustc_target::abi::{Abi, Primitive};
+use rustc_target::abi::{Abi, FieldIdx, Primitive};
 
 use crate::build::expr::as_place::PlaceBase;
 use crate::build::expr::category::{Category, RvalueFunc};
 use crate::build::{BlockAnd, BlockAndExtension, Builder, NeedsTemporary};
 use rustc_hir::lang_items::LangItem;
 use rustc_middle::middle::region;
+use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::AssertKind;
 use rustc_middle::mir::Place;
 use rustc_middle::mir::*;
@@ -63,7 +64,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             block,
                             scope,
                             &this.thir[value],
-                            None,
+                            LocalInfo::Boring,
                             NeedsTemporary::No
                         )
                     );
@@ -72,19 +73,34 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
             ExprKind::Binary { op, lhs, rhs } => {
                 let lhs = unpack!(
-                    block =
-                        this.as_operand(block, scope, &this.thir[lhs], None, NeedsTemporary::Maybe)
+                    block = this.as_operand(
+                        block,
+                        scope,
+                        &this.thir[lhs],
+                        LocalInfo::Boring,
+                        NeedsTemporary::Maybe
+                    )
                 );
                 let rhs = unpack!(
-                    block =
-                        this.as_operand(block, scope, &this.thir[rhs], None, NeedsTemporary::No)
+                    block = this.as_operand(
+                        block,
+                        scope,
+                        &this.thir[rhs],
+                        LocalInfo::Boring,
+                        NeedsTemporary::No
+                    )
                 );
                 this.build_binary_op(block, op, expr_span, expr.ty, lhs, rhs)
             }
             ExprKind::Unary { op, arg } => {
                 let arg = unpack!(
-                    block =
-                        this.as_operand(block, scope, &this.thir[arg], None, NeedsTemporary::No)
+                    block = this.as_operand(
+                        block,
+                        scope,
+                        &this.thir[arg],
+                        LocalInfo::Boring,
+                        NeedsTemporary::No
+                    )
                 );
                 // Check for -MIN on signed integers
                 if this.check_overflow && op == UnOp::Neg && expr.ty.is_signed() {
@@ -155,7 +171,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         args: vec![Operand::Move(size), Operand::Move(align)],
                         destination: storage,
                         target: Some(success),
-                        cleanup: None,
+                        unwind: UnwindAction::Continue,
                         from_hir_call: false,
                         fn_span: expr_span,
                     },
@@ -259,7 +275,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 } else {
                     let ty = source.ty;
                     let source = unpack!(
-                        block = this.as_operand(block, scope, source, None, NeedsTemporary::No)
+                        block = this.as_operand(block, scope, source, LocalInfo::Boring, NeedsTemporary::No)
                     );
                     (source, ty)
                 };
@@ -271,8 +287,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
             ExprKind::Pointer { cast, source } => {
                 let source = unpack!(
-                    block =
-                        this.as_operand(block, scope, &this.thir[source], None, NeedsTemporary::No)
+                    block = this.as_operand(
+                        block,
+                        scope,
+                        &this.thir[source],
+                        LocalInfo::Boring,
+                        NeedsTemporary::No
+                    )
                 );
                 block.and(Rvalue::Cast(CastKind::Pointer(cast), source, expr.ty))
             }
@@ -305,7 +326,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 // first process the set of fields
                 let el_ty = expr.ty.sequence_element_type(this.tcx);
-                let fields: Vec<_> = fields
+                let fields: IndexVec<FieldIdx, _> = fields
                     .into_iter()
                     .copied()
                     .map(|f| {
@@ -314,7 +335,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                 block,
                                 scope,
                                 &this.thir[f],
-                                None,
+                                LocalInfo::Boring,
                                 NeedsTemporary::Maybe
                             )
                         )
@@ -326,7 +347,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ExprKind::Tuple { ref fields } => {
                 // see (*) above
                 // first process the set of fields
-                let fields: Vec<_> = fields
+                let fields: IndexVec<FieldIdx, _> = fields
                     .into_iter()
                     .copied()
                     .map(|f| {
@@ -335,7 +356,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                 block,
                                 scope,
                                 &this.thir[f],
-                                None,
+                                LocalInfo::Boring,
                                 NeedsTemporary::Maybe
                             )
                         )
@@ -380,7 +401,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 }
 
                 // see (*) above
-                let operands: Vec<_> = upvars
+                let operands: IndexVec<FieldIdx, _> = upvars
                     .into_iter()
                     .copied()
                     .map(|upvar| {
@@ -423,7 +444,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                                 block,
                                                 scope,
                                                 upvar,
-                                                None,
+                                                LocalInfo::Boring,
                                                 NeedsTemporary::Maybe
                                             )
                                         )
@@ -501,8 +522,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     Category::of(&expr.kind),
                     Some(Category::Rvalue(RvalueFunc::AsRvalue) | Category::Constant)
                 ));
-                let operand =
-                    unpack!(block = this.as_operand(block, scope, expr, None, NeedsTemporary::No));
+                let operand = unpack!(
+                    block =
+                        this.as_operand(block, scope, expr, LocalInfo::Boring, NeedsTemporary::No)
+                );
                 block.and(Rvalue::Use(operand))
             }
         }
@@ -519,30 +542,78 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ) -> BlockAnd<Rvalue<'tcx>> {
         let source_info = self.source_info(span);
         let bool_ty = self.tcx.types.bool;
-        if self.check_overflow && op.is_checkable() && ty.is_integral() {
-            let result_tup = self.tcx.mk_tup(&[ty, bool_ty]);
-            let result_value = self.temp(result_tup, span);
-
-            self.cfg.push_assign(
-                block,
-                source_info,
-                result_value,
-                Rvalue::CheckedBinaryOp(op, Box::new((lhs.to_copy(), rhs.to_copy()))),
-            );
-            let val_fld = Field::new(0);
-            let of_fld = Field::new(1);
+        let rvalue = match op {
+            BinOp::Add | BinOp::Sub | BinOp::Mul if self.check_overflow && ty.is_integral() => {
+                let result_tup = self.tcx.mk_tup(&[ty, bool_ty]);
+                let result_value = self.temp(result_tup, span);
 
-            let tcx = self.tcx;
-            let val = tcx.mk_place_field(result_value, val_fld, ty);
-            let of = tcx.mk_place_field(result_value, of_fld, bool_ty);
+                self.cfg.push_assign(
+                    block,
+                    source_info,
+                    result_value,
+                    Rvalue::CheckedBinaryOp(op, Box::new((lhs.to_copy(), rhs.to_copy()))),
+                );
+                let val_fld = FieldIdx::new(0);
+                let of_fld = FieldIdx::new(1);
 
-            let err = AssertKind::Overflow(op, lhs, rhs);
+                let tcx = self.tcx;
+                let val = tcx.mk_place_field(result_value, val_fld, ty);
+                let of = tcx.mk_place_field(result_value, of_fld, bool_ty);
 
-            block = self.assert(block, Operand::Move(of), false, err, span);
+                let err = AssertKind::Overflow(op, lhs, rhs);
+                block = self.assert(block, Operand::Move(of), false, err, span);
 
-            block.and(Rvalue::Use(Operand::Move(val)))
-        } else {
-            if ty.is_integral() && (op == BinOp::Div || op == BinOp::Rem) {
+                Rvalue::Use(Operand::Move(val))
+            }
+            BinOp::Shl | BinOp::Shr if self.check_overflow && ty.is_integral() => {
+                // For an unsigned RHS, the shift is in-range for `rhs < bits`.
+                // For a signed RHS, `IntToInt` cast to the equivalent unsigned
+                // type and do that same comparison. Because the type is the
+                // same size, there's no negative shift amount that ends up
+                // overlapping with valid ones, thus it catches negatives too.
+                let (lhs_size, _) = ty.int_size_and_signed(self.tcx);
+                let rhs_ty = rhs.ty(&self.local_decls, self.tcx);
+                let (rhs_size, _) = rhs_ty.int_size_and_signed(self.tcx);
+
+                let (unsigned_rhs, unsigned_ty) = match rhs_ty.kind() {
+                    ty::Uint(_) => (rhs.to_copy(), rhs_ty),
+                    ty::Int(int_width) => {
+                        let uint_ty = self.tcx.mk_mach_uint(int_width.to_unsigned());
+                        let rhs_temp = self.temp(uint_ty, span);
+                        self.cfg.push_assign(
+                            block,
+                            source_info,
+                            rhs_temp,
+                            Rvalue::Cast(CastKind::IntToInt, rhs.to_copy(), uint_ty),
+                        );
+                        (Operand::Move(rhs_temp), uint_ty)
+                    }
+                    _ => unreachable!("only integers are shiftable"),
+                };
+
+                // This can't overflow because the largest shiftable types are 128-bit,
+                // which fits in `u8`, the smallest possible `unsigned_ty`.
+                // (And `from_uint` will `bug!` if that's ever no longer true.)
+                let lhs_bits = Operand::const_from_scalar(
+                    self.tcx,
+                    unsigned_ty,
+                    Scalar::from_uint(lhs_size.bits(), rhs_size),
+                    span,
+                );
+
+                let inbounds = self.temp(bool_ty, span);
+                self.cfg.push_assign(
+                    block,
+                    source_info,
+                    inbounds,
+                    Rvalue::BinaryOp(BinOp::Lt, Box::new((unsigned_rhs, lhs_bits))),
+                );
+
+                let overflow_err = AssertKind::Overflow(op, lhs.to_copy(), rhs.to_copy());
+                block = self.assert(block, Operand::Move(inbounds), true, overflow_err, span);
+                Rvalue::BinaryOp(op, Box::new((lhs, rhs)))
+            }
+            BinOp::Div | BinOp::Rem if ty.is_integral() => {
                 // Checking division and remainder is more complex, since we 1. always check
                 // and 2. there are two possible failure cases, divide-by-zero and overflow.
 
@@ -601,10 +672,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                     block = self.assert(block, Operand::Move(of), false, overflow_err, span);
                 }
-            }
 
-            block.and(Rvalue::BinaryOp(op, Box::new((lhs, rhs))))
-        }
+                Rvalue::BinaryOp(op, Box::new((lhs, rhs)))
+            }
+            _ => Rvalue::BinaryOp(op, Box::new((lhs, rhs))),
+        };
+        block.and(rvalue)
     }
 
     fn build_zero_repeat(
@@ -621,21 +694,26 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // Repeating a const does nothing
         } else {
             // For a non-const, we may need to generate an appropriate `Drop`
-            let value_operand =
-                unpack!(block = this.as_operand(block, scope, value, None, NeedsTemporary::No));
+            let value_operand = unpack!(
+                block = this.as_operand(block, scope, value, LocalInfo::Boring, NeedsTemporary::No)
+            );
             if let Operand::Move(to_drop) = value_operand {
                 let success = this.cfg.start_new_block();
                 this.cfg.terminate(
                     block,
                     outer_source_info,
-                    TerminatorKind::Drop { place: to_drop, target: success, unwind: None },
+                    TerminatorKind::Drop {
+                        place: to_drop,
+                        target: success,
+                        unwind: UnwindAction::Continue,
+                    },
                 );
                 this.diverge_from(block);
                 block = success;
             }
             this.record_operands_moved(&[value_operand]);
         }
-        block.and(Rvalue::Aggregate(Box::new(AggregateKind::Array(elem_ty)), Vec::new()))
+        block.and(Rvalue::Aggregate(Box::new(AggregateKind::Array(elem_ty)), IndexVec::new()))
     }
 
     fn limit_capture_mutability(
diff --git a/compiler/rustc_mir_build/src/build/expr/as_temp.rs b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
index 3d3cf7555..c8910c272 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_temp.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
@@ -49,29 +49,28 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             debug!("creating temp {:?} with block_context: {:?}", local_decl, this.block_context);
-            // Find out whether this temp is being created within the
-            // tail expression of a block whose result is ignored.
-            if let Some(tail_info) = this.block_context.currently_in_block_tail() {
-                local_decl = local_decl.block_tail(tail_info);
-            }
-            match expr.kind {
+            let local_info = match expr.kind {
                 ExprKind::StaticRef { def_id, .. } => {
                     assert!(!this.tcx.is_thread_local_static(def_id));
                     local_decl.internal = true;
-                    local_decl.local_info =
-                        Some(Box::new(LocalInfo::StaticRef { def_id, is_thread_local: false }));
+                    LocalInfo::StaticRef { def_id, is_thread_local: false }
                 }
                 ExprKind::ThreadLocalRef(def_id) => {
                     assert!(this.tcx.is_thread_local_static(def_id));
                     local_decl.internal = true;
-                    local_decl.local_info =
-                        Some(Box::new(LocalInfo::StaticRef { def_id, is_thread_local: true }));
+                    LocalInfo::StaticRef { def_id, is_thread_local: true }
                 }
                 ExprKind::NamedConst { def_id, .. } | ExprKind::ConstParam { def_id, .. } => {
-                    local_decl.local_info = Some(Box::new(LocalInfo::ConstRef { def_id }));
+                    LocalInfo::ConstRef { def_id }
                 }
-                _ => {}
-            }
+                // Find out whether this temp is being created within the
+                // tail expression of a block whose result is ignored.
+                _ if let Some(tail_info) = this.block_context.currently_in_block_tail() => {
+                    LocalInfo::BlockTailTemp(tail_info)
+                }
+                _ => LocalInfo::Boring,
+            };
+            **local_decl.local_info.as_mut().assert_crate_local() = local_info;
             this.local_decls.push(local_decl)
         };
         let temp_place = Place::from(temp);
diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs
index dac9bf0a8..05a723a6b 100644
--- a/compiler/rustc_mir_build/src/build/expr/into.rs
+++ b/compiler/rustc_mir_build/src/build/expr/into.rs
@@ -6,7 +6,6 @@ use rustc_ast::InlineAsmOptions;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
-use rustc_index::vec::Idx;
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty::CanonicalUserTypeAnnotation;
@@ -229,7 +228,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     this.cfg.terminate(
                         loop_block,
                         source_info,
-                        TerminatorKind::FalseUnwind { real_target: body_block, unwind: None },
+                        TerminatorKind::FalseUnwind {
+                            real_target: body_block,
+                            unwind: UnwindAction::Continue,
+                        },
                     );
                     this.diverge_from(loop_block);
 
@@ -265,7 +267,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     TerminatorKind::Call {
                         func: fun,
                         args,
-                        cleanup: None,
+                        unwind: UnwindAction::Continue,
                         destination,
                         // The presence or absence of a return edge affects control-flow sensitive
                         // MIR checks and ultimately whether code is accepted or not. We can only
@@ -319,7 +321,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // See the notes for `ExprKind::Array` in `as_rvalue` and for
                 // `ExprKind::Borrow` above.
                 let is_union = adt_def.is_union();
-                let active_field_index = is_union.then(|| fields[0].name.index());
+                let active_field_index = is_union.then(|| fields[0].name);
 
                 let scope = this.local_scope();
 
@@ -328,7 +330,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let fields_map: FxHashMap<_, _> = fields
                     .into_iter()
                     .map(|f| {
-                        let local_info = Box::new(LocalInfo::AggregateTemp);
                         (
                             f.name,
                             unpack!(
@@ -336,7 +337,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                     block,
                                     Some(scope),
                                     &this.thir[f.expr],
-                                    Some(local_info),
+                                    LocalInfo::AggregateTemp,
                                     NeedsTemporary::Maybe,
                                 )
                             ),
@@ -344,10 +345,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     })
                     .collect();
 
-                let field_names: Vec<_> =
-                    (0..adt_def.variant(variant_index).fields.len()).map(Field::new).collect();
+                let field_names = adt_def.variant(variant_index).fields.indices();
 
-                let fields: Vec<_> = if let Some(FruInfo { base, field_types }) = base {
+                let fields = if let Some(FruInfo { base, field_types }) = base {
                     let place_builder =
                         unpack!(block = this.as_place_builder(block, &this.thir[*base]));
 
@@ -364,7 +364,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         })
                         .collect()
                 } else {
-                    field_names.iter().filter_map(|n| fields_map.get(n).cloned()).collect()
+                    field_names.filter_map(|n| fields_map.get(&n).cloned()).collect()
                 };
 
                 let inferred_ty = expr.ty;
@@ -469,7 +469,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         } else {
                             Some(destination_block)
                         },
-                        cleanup: None,
+                        unwind: if options.contains(InlineAsmOptions::MAY_UNWIND) {
+                            UnwindAction::Continue
+                        } else {
+                            UnwindAction::Unreachable
+                        },
                     },
                 );
                 if options.contains(InlineAsmOptions::MAY_UNWIND) {
@@ -526,7 +530,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         block,
                         Some(scope),
                         &this.thir[value],
-                        None,
+                        LocalInfo::Boring,
                         NeedsTemporary::No
                     )
                 );
diff --git a/compiler/rustc_mir_build/src/build/expr/stmt.rs b/compiler/rustc_mir_build/src/build/expr/stmt.rs
index 780836851..ea5aeb67d 100644
--- a/compiler/rustc_mir_build/src/build/expr/stmt.rs
+++ b/compiler/rustc_mir_build/src/build/expr/stmt.rs
@@ -40,7 +40,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // Generate better code for things that don't need to be
                 // dropped.
                 if lhs.ty.needs_drop(this.tcx, this.param_env) {
-                    let rhs = unpack!(block = this.as_local_operand(block, rhs));
+                    let rhs = unpack!(block = this.as_local_rvalue(block, rhs));
                     let lhs = unpack!(block = this.as_place(block, lhs));
                     unpack!(block = this.build_drop_and_replace(block, lhs_span, lhs, rhs));
                 } else {
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index de2851a1a..4926ff85d 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -556,6 +556,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             _ => {
                 let place_builder = unpack!(block = self.as_place_builder(block, initializer));
+
+                if let Some(place) = place_builder.try_to_place(self) {
+                    let source_info = self.source_info(initializer.span);
+                    self.cfg.push_place_mention(block, source_info, place);
+                }
+
                 self.place_into_pattern(block, &irrefutable_pat, place_builder, true)
             }
         }
@@ -576,13 +582,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             false,
             &mut [&mut candidate],
         );
+
         // For matches and function arguments, the place that is being matched
         // can be set when creating the variables. But the place for
         // let PATTERN = ... might not even exist until we do the assignment.
         // so we set it here instead.
         if set_match_place {
-            let mut candidate_ref = &candidate;
-            while let Some(next) = {
+            let mut next = Some(&candidate);
+            while let Some(candidate_ref) = next.take() {
                 for binding in &candidate_ref.bindings {
                     let local = self.var_local_id(binding.var_id, OutsideGuard);
                     // `try_to_place` may fail if it is unable to resolve the given
@@ -600,9 +607,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // };
                     // ```
                     if let Some(place) = initializer.try_to_place(self) {
-                        let Some(box LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
+                        let LocalInfo::User(BindingForm::Var(
                             VarBindingForm { opt_match_place: Some((ref mut match_place, _)), .. },
-                        )))) = self.local_decls[local].local_info else {
+                        )) = **self.local_decls[local].local_info.as_mut().assert_crate_local() else {
                             bug!("Let binding to non-user variable.")
                         };
                         *match_place = Some(place);
@@ -610,9 +617,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 }
                 // All of the subcandidates should bind the same locals, so we
                 // only visit the first one.
-                candidate_ref.subcandidates.get(0)
-            } {
-                candidate_ref = next;
+                next = candidate_ref.subcandidates.get(0)
             }
         }
 
@@ -1749,7 +1754,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let fake_borrow_ty = tcx.mk_imm_ref(tcx.lifetimes.re_erased, fake_borrow_deref_ty);
                 let mut fake_borrow_temp = LocalDecl::new(fake_borrow_ty, temp_span);
                 fake_borrow_temp.internal = self.local_decls[matched_place.local].internal;
-                fake_borrow_temp.local_info = Some(Box::new(LocalInfo::FakeBorrow));
+                fake_borrow_temp.local_info = ClearCrossCrate::Set(Box::new(LocalInfo::FakeBorrow));
                 let fake_borrow_temp = self.local_decls.push(fake_borrow_temp);
 
                 (matched_place, fake_borrow_temp)
@@ -1881,6 +1886,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // let place = Foo::new();
         // match place { Foo { .. } if { let tmp1 = &place; inspect(*tmp1) }
         //     => { let tmp2 = place; feed(tmp2) }, ... }
+        // ```
         //
         // And an input like:
         //
@@ -2218,8 +2224,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             user_ty: if user_ty.is_empty() { None } else { Some(Box::new(user_ty)) },
             source_info,
             internal: false,
-            is_block_tail: None,
-            local_info: Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
+            local_info: ClearCrossCrate::Set(Box::new(LocalInfo::User(BindingForm::Var(
                 VarBindingForm {
                     binding_mode,
                     // hypothetically, `visit_primary_bindings` could try to unzip
@@ -2230,13 +2235,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     opt_match_place,
                     pat_span,
                 },
-            ))))),
+            )))),
         };
         let for_arm_body = self.local_decls.push(local);
         self.var_debug_info.push(VarDebugInfo {
             name,
             source_info: debug_source_info,
             value: VarDebugInfoContents::Place(for_arm_body.into()),
+            argument_index: None,
         });
         let locals = if has_guard.0 {
             let ref_for_guard = self.local_decls.push(LocalDecl::<'tcx> {
@@ -2247,15 +2253,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 user_ty: None,
                 source_info,
                 internal: false,
-                is_block_tail: None,
-                local_info: Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(
+                local_info: ClearCrossCrate::Set(Box::new(LocalInfo::User(
                     BindingForm::RefForGuard,
-                )))),
+                ))),
             });
             self.var_debug_info.push(VarDebugInfo {
                 name,
                 source_info: debug_source_info,
                 value: VarDebugInfoContents::Place(ref_for_guard.into()),
+                argument_index: None,
             });
             LocalsForNode::ForGuard { ref_for_guard, for_arm_body }
         } else {
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index 2de89f67d..8a03ea7e2 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -263,7 +263,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             args: vec![Operand::Move(ref_string)],
                             destination: ref_str,
                             target: Some(eq_block),
-                            cleanup: None,
+                            unwind: UnwindAction::Continue,
                             from_hir_call: false,
                             fn_span: source_info.span
                         }
@@ -466,7 +466,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 args: vec![val, expect],
                 destination: eq_result,
                 target: Some(eq_block),
-                cleanup: None,
+                unwind: UnwindAction::Continue,
                 from_hir_call: false,
                 fn_span: source_info.span,
             },
diff --git a/compiler/rustc_mir_build/src/build/misc.rs b/compiler/rustc_mir_build/src/build/misc.rs
index baeb2718c..90d78658f 100644
--- a/compiler/rustc_mir_build/src/build/misc.rs
+++ b/compiler/rustc_mir_build/src/build/misc.rs
@@ -5,7 +5,7 @@ use crate::build::Builder;
 
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty};
-use rustc_span::{Span, DUMMY_SP};
+use rustc_span::Span;
 use rustc_trait_selection::infer::InferCtxtExt;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
@@ -66,7 +66,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     pub(crate) fn consume_by_copy_or_move(&self, place: Place<'tcx>) -> Operand<'tcx> {
         let tcx = self.tcx;
         let ty = place.ty(&self.local_decls, tcx).ty;
-        if !self.infcx.type_is_copy_modulo_regions(self.param_env, ty, DUMMY_SP) {
+        if !self.infcx.type_is_copy_modulo_regions(self.param_env, ty) {
             Operand::Move(place)
         } else {
             Operand::Copy(place)
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
index b3f9d8282..bc50bcbc3 100644
--- a/compiler/rustc_mir_build/src/build/mod.rs
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -3,6 +3,7 @@ use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::scope::DropKind;
 use rustc_apfloat::ieee::{Double, Single};
 use rustc_apfloat::Float;
+use rustc_ast::attr;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sorted_map::SortedIndexMultiMap;
 use rustc_errors::ErrorGuaranteed;
@@ -10,7 +11,7 @@ use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::{GeneratorKind, Node};
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{Idx, IndexSlice, IndexVec};
 use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
 use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
 use rustc_middle::middle::region;
@@ -24,6 +25,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
 use rustc_span::Symbol;
+use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 
 use super::lints;
@@ -48,17 +50,15 @@ pub(crate) fn mir_built(
 /// Construct the MIR for a given `DefId`.
 fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> {
     // Ensure unsafeck and abstract const building is ran before we steal the THIR.
-    // We can't use `ensure()` for `thir_abstract_const` as it doesn't compute the query
-    // if inputs are green. This can cause ICEs when calling `thir_abstract_const` after
-    // THIR has been stolen if we haven't computed this query yet.
     match def {
         ty::WithOptConstParam { did, const_param_did: Some(const_param_did) } => {
-            tcx.ensure().thir_check_unsafety_for_const_arg((did, const_param_did));
-            drop(tcx.thir_abstract_const_of_const_arg((did, const_param_did)));
+            tcx.ensure_with_value().thir_check_unsafety_for_const_arg((did, const_param_did));
+            tcx.ensure_with_value().thir_abstract_const_of_const_arg((did, const_param_did));
         }
         ty::WithOptConstParam { did, const_param_did: None } => {
-            tcx.ensure().thir_check_unsafety(did);
-            drop(tcx.thir_abstract_const(did));
+            tcx.ensure_with_value().thir_check_unsafety(did);
+            tcx.ensure_with_value().thir_abstract_const(did);
+            tcx.ensure_with_value().check_match(did);
         }
     }
 
@@ -683,7 +683,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // Some functions always have overflow checks enabled,
         // however, they may not get codegen'd, depending on
         // the settings for the crate they are codegened in.
-        let mut check_overflow = tcx.sess.contains_name(attrs, sym::rustc_inherit_overflow_checks);
+        let mut check_overflow = attr::contains_name(attrs, sym::rustc_inherit_overflow_checks);
         // Respect -C overflow-checks.
         check_overflow |= tcx.sess.overflow_checks();
         // Constants always need overflow checks.
@@ -795,7 +795,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let mutability = captured_place.mutability;
 
                 let mut projs = closure_env_projs.clone();
-                projs.push(ProjectionElem::Field(Field::new(i), ty));
+                projs.push(ProjectionElem::Field(FieldIdx::new(i), ty));
                 match capture {
                     ty::UpvarCapture::ByValue => {}
                     ty::UpvarCapture::ByRef(..) => {
@@ -811,6 +811,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     name,
                     source_info: SourceInfo::outermost(captured_place.var_ident.span),
                     value: VarDebugInfoContents::Place(use_place),
+                    argument_index: None,
                 });
 
                 let capture = Capture { captured_place, use_place, mutability };
@@ -822,12 +823,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn args_and_body(
         &mut self,
         mut block: BasicBlock,
-        arguments: &IndexVec<ParamId, Param<'tcx>>,
+        arguments: &IndexSlice<ParamId, Param<'tcx>>,
         argument_scope: region::Scope,
         expr: &Expr<'tcx>,
     ) -> BlockAnd<()> {
         // Allocate locals for the function arguments
-        for param in arguments.iter() {
+        for (argument_index, param) in arguments.iter().enumerate() {
             let source_info =
                 SourceInfo::outermost(param.pat.as_ref().map_or(self.fn_span, |pat| pat.span));
             let arg_local =
@@ -839,6 +840,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     name,
                     source_info,
                     value: VarDebugInfoContents::Place(arg_local.into()),
+                    argument_index: Some(argument_index as u16 + 1),
                 });
             }
         }
@@ -879,21 +881,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 } => {
                     self.local_decls[local].mutability = mutability;
                     self.local_decls[local].source_info.scope = self.source_scope;
-                    self.local_decls[local].local_info = if let Some(kind) = param.self_kind {
-                        Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(
-                            BindingForm::ImplicitSelf(kind),
-                        ))))
-                    } else {
-                        let binding_mode = ty::BindingMode::BindByValue(mutability);
-                        Some(Box::new(LocalInfo::User(ClearCrossCrate::Set(BindingForm::Var(
-                            VarBindingForm {
+                    **self.local_decls[local].local_info.as_mut().assert_crate_local() =
+                        if let Some(kind) = param.self_kind {
+                            LocalInfo::User(BindingForm::ImplicitSelf(kind))
+                        } else {
+                            let binding_mode = ty::BindingMode::BindByValue(mutability);
+                            LocalInfo::User(BindingForm::Var(VarBindingForm {
                                 binding_mode,
                                 opt_ty_info: param.ty_span,
                                 opt_match_place: Some((None, span)),
                                 pat_span: span,
-                            },
-                        )))))
-                    };
+                            }))
+                        };
                     self.var_indices.insert(var, LocalsForNode::One(local));
                 }
                 _ => {
diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
index 591b41633..f32d2db4e 100644
--- a/compiler/rustc_mir_build/src/build/scope.rs
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -86,12 +86,12 @@ use std::mem;
 use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder, CFG};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::HirId;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
 use rustc_middle::thir::{Expr, LintLevel};
 
-use rustc_span::{Span, DUMMY_SP};
+use rustc_span::{DesugaringKind, Span, DUMMY_SP};
 
 #[derive(Debug)]
 pub struct Scopes<'tcx> {
@@ -360,7 +360,7 @@ impl DropTree {
     fn link_blocks<'tcx>(
         &self,
         cfg: &mut CFG<'tcx>,
-        blocks: &IndexVec<DropIdx, Option<BasicBlock>>,
+        blocks: &IndexSlice<DropIdx, Option<BasicBlock>>,
     ) {
         for (drop_idx, drop_data) in self.drops.iter_enumerated().rev() {
             let Some(block) = blocks[drop_idx] else { continue };
@@ -369,7 +369,7 @@ impl DropTree {
                     let terminator = TerminatorKind::Drop {
                         target: blocks[drop_data.1].unwrap(),
                         // The caller will handle this if needed.
-                        unwind: None,
+                        unwind: UnwindAction::Terminate,
                         place: drop_data.0.local.into(),
                     };
                     cfg.terminate(block, drop_data.0.source_info, terminator);
@@ -1072,7 +1072,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 TerminatorKind::Assert { .. }
                     | TerminatorKind::Call { .. }
                     | TerminatorKind::Drop { .. }
-                    | TerminatorKind::DropAndReplace { .. }
                     | TerminatorKind::FalseUnwind { .. }
                     | TerminatorKind::InlineAsm { .. }
             ),
@@ -1118,24 +1117,39 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 
     /// Utility function for *non*-scope code to build their own drops
+    /// Force a drop at this point in the MIR by creating a new block.
     pub(crate) fn build_drop_and_replace(
         &mut self,
         block: BasicBlock,
         span: Span,
         place: Place<'tcx>,
-        value: Operand<'tcx>,
+        value: Rvalue<'tcx>,
     ) -> BlockAnd<()> {
+        let span = self.tcx.with_stable_hashing_context(|hcx| {
+            span.mark_with_reason(None, DesugaringKind::Replace, self.tcx.sess.edition(), hcx)
+        });
         let source_info = self.source_info(span);
-        let next_target = self.cfg.start_new_block();
+
+        // create the new block for the assignment
+        let assign = self.cfg.start_new_block();
+        self.cfg.push_assign(assign, source_info, place, value.clone());
+
+        // create the new block for the assignment in the case of unwinding
+        let assign_unwind = self.cfg.start_new_cleanup_block();
+        self.cfg.push_assign(assign_unwind, source_info, place, value.clone());
 
         self.cfg.terminate(
             block,
             source_info,
-            TerminatorKind::DropAndReplace { place, value, target: next_target, unwind: None },
+            TerminatorKind::Drop {
+                place,
+                target: assign,
+                unwind: UnwindAction::Cleanup(assign_unwind),
+            },
         );
         self.diverge_from(block);
 
-        next_target.unit()
+        assign.unit()
     }
 
     /// Creates an `Assert` terminator and return the success block.
@@ -1155,7 +1169,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         self.cfg.terminate(
             block,
             source_info,
-            TerminatorKind::Assert { cond, expected, msg, target: success_block, cleanup: None },
+            TerminatorKind::Assert {
+                cond,
+                expected,
+                msg,
+                target: success_block,
+                unwind: UnwindAction::Continue,
+            },
         );
         self.diverge_from(block);
 
@@ -1234,7 +1254,11 @@ fn build_scope_drops<'tcx>(
                 cfg.terminate(
                     block,
                     source_info,
-                    TerminatorKind::Drop { place: local.into(), target: next, unwind: None },
+                    TerminatorKind::Drop {
+                        place: local.into(),
+                        target: next,
+                        unwind: UnwindAction::Continue,
+                    },
                 );
                 block = next;
             }
@@ -1413,18 +1437,24 @@ impl<'tcx> DropTreeBuilder<'tcx> for Unwind {
     fn add_entry(cfg: &mut CFG<'tcx>, from: BasicBlock, to: BasicBlock) {
         let term = &mut cfg.block_data_mut(from).terminator_mut();
         match &mut term.kind {
-            TerminatorKind::Drop { unwind, .. }
-            | TerminatorKind::DropAndReplace { unwind, .. }
-            | TerminatorKind::FalseUnwind { unwind, .. }
-            | TerminatorKind::Call { cleanup: unwind, .. }
-            | TerminatorKind::Assert { cleanup: unwind, .. }
-            | TerminatorKind::InlineAsm { cleanup: unwind, .. } => {
-                *unwind = Some(to);
+            TerminatorKind::Drop { unwind, .. } => {
+                if let UnwindAction::Cleanup(unwind) = *unwind {
+                    let source_info = term.source_info;
+                    cfg.terminate(unwind, source_info, TerminatorKind::Goto { target: to });
+                } else {
+                    *unwind = UnwindAction::Cleanup(to);
+                }
+            }
+            TerminatorKind::FalseUnwind { unwind, .. }
+            | TerminatorKind::Call { unwind, .. }
+            | TerminatorKind::Assert { unwind, .. }
+            | TerminatorKind::InlineAsm { unwind, .. } => {
+                *unwind = UnwindAction::Cleanup(to);
             }
             TerminatorKind::Goto { .. }
             | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::Yield { .. }
diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index dc4d2276e..43e787db4 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -6,11 +6,11 @@ use rustc_errors::{
     error_code, AddToDiagnostic, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed,
     Handler, IntoDiagnostic, MultiSpan, SubdiagnosticMessage,
 };
-use rustc_hir::def::Res;
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::thir::Pat;
 use rustc_middle::ty::{self, Ty};
-use rustc_span::{symbol::Ident, Span};
+use rustc_span::symbol::Symbol;
+use rustc_span::Span;
 
 #[derive(LintDiagnostic)]
 #[diag(mir_build_unconditional_recursion)]
@@ -534,18 +534,10 @@ pub struct TrailingIrrefutableLetPatterns {
 #[derive(LintDiagnostic)]
 #[diag(mir_build_bindings_with_variant_name, code = "E0170")]
 pub struct BindingsWithVariantName {
-    #[suggestion(code = "{ty_path}::{ident}", applicability = "machine-applicable")]
+    #[suggestion(code = "{ty_path}::{name}", applicability = "machine-applicable")]
     pub suggestion: Option<Span>,
     pub ty_path: String,
-    pub ident: Ident,
-}
-
-#[derive(LintDiagnostic)]
-#[diag(mir_build_irrefutable_let_patterns_generic_let)]
-#[note]
-#[help]
-pub struct IrrefutableLetPatternsGenericLet {
-    pub count: usize,
+    pub name: Symbol,
 }
 
 #[derive(LintDiagnostic)]
@@ -584,13 +576,12 @@ pub struct IrrefutableLetPatternsWhileLet {
 #[diag(mir_build_borrow_of_moved_value)]
 pub struct BorrowOfMovedValue<'tcx> {
     #[primary_span]
-    pub span: Span,
     #[label]
     #[label(mir_build_occurs_because_label)]
     pub binding_span: Span,
     #[label(mir_build_value_borrowed_label)]
     pub conflicts_ref: Vec<Span>,
-    pub name: Ident,
+    pub name: Symbol,
     pub ty: Ty<'tcx>,
     #[suggestion(code = "ref ", applicability = "machine-applicable")]
     pub suggest_borrowing: Option<Span>,
@@ -602,7 +593,7 @@ pub struct MultipleMutBorrows {
     #[primary_span]
     pub span: Span,
     #[subdiagnostic]
-    pub occurences: Vec<Conflict>,
+    pub occurrences: Vec<Conflict>,
 }
 
 #[derive(Diagnostic)]
@@ -611,7 +602,7 @@ pub struct AlreadyBorrowed {
     #[primary_span]
     pub span: Span,
     #[subdiagnostic]
-    pub occurences: Vec<Conflict>,
+    pub occurrences: Vec<Conflict>,
 }
 
 #[derive(Diagnostic)]
@@ -620,7 +611,7 @@ pub struct AlreadyMutBorrowed {
     #[primary_span]
     pub span: Span,
     #[subdiagnostic]
-    pub occurences: Vec<Conflict>,
+    pub occurrences: Vec<Conflict>,
 }
 
 #[derive(Diagnostic)]
@@ -629,7 +620,7 @@ pub struct MovedWhileBorrowed {
     #[primary_span]
     pub span: Span,
     #[subdiagnostic]
-    pub occurences: Vec<Conflict>,
+    pub occurrences: Vec<Conflict>,
 }
 
 #[derive(Subdiagnostic)]
@@ -638,19 +629,19 @@ pub enum Conflict {
     Mut {
         #[primary_span]
         span: Span,
-        name: Ident,
+        name: Symbol,
     },
     #[label(mir_build_borrow)]
     Ref {
         #[primary_span]
         span: Span,
-        name: Ident,
+        name: Symbol,
     },
     #[label(mir_build_moved)]
     Moved {
         #[primary_span]
         span: Span,
-        name: Ident,
+        name: Symbol,
     },
 }
 
@@ -663,6 +654,8 @@ pub struct UnionPattern {
 
 #[derive(Diagnostic)]
 #[diag(mir_build_type_not_structural)]
+#[note(mir_build_type_not_structural_tip)]
+#[note(mir_build_type_not_structural_more_info)]
 pub struct TypeNotStructural<'tcx> {
     #[primary_span]
     pub span: Span,
@@ -695,12 +688,16 @@ pub struct PointerPattern;
 
 #[derive(LintDiagnostic)]
 #[diag(mir_build_indirect_structural_match)]
+#[note(mir_build_type_not_structural_tip)]
+#[note(mir_build_type_not_structural_more_info)]
 pub struct IndirectStructuralMatch<'tcx> {
     pub non_sm_ty: Ty<'tcx>,
 }
 
 #[derive(LintDiagnostic)]
 #[diag(mir_build_nontrivial_structural_match)]
+#[note(mir_build_type_not_structural_tip)]
+#[note(mir_build_type_not_structural_more_info)]
 pub struct NontrivialStructuralMatch<'tcx> {
     pub non_sm_ty: Ty<'tcx>,
 }
@@ -796,8 +793,6 @@ pub(crate) struct PatternNotCovered<'s, 'tcx> {
     pub let_suggestion: Option<SuggestLet>,
     #[subdiagnostic]
     pub misc_suggestion: Option<MiscPatternSuggestion>,
-    #[subdiagnostic]
-    pub res_defined_here: Option<ResDefinedHere>,
 }
 
 #[derive(Subdiagnostic)]
@@ -832,14 +827,6 @@ impl<'tcx> AddToDiagnostic for AdtDefinedHere<'tcx> {
 }
 
 #[derive(Subdiagnostic)]
-#[label(mir_build_res_defined_here)]
-pub struct ResDefinedHere {
-    #[primary_span]
-    pub def_span: Span,
-    pub res: Res,
-}
-
-#[derive(Subdiagnostic)]
 #[suggestion(
     mir_build_interpreted_as_const,
     code = "{variable}_var",
@@ -849,9 +836,7 @@ pub struct ResDefinedHere {
 pub struct InterpretedAsConst {
     #[primary_span]
     pub span: Span,
-    pub article: &'static str,
     pub variable: String,
-    pub res: Res,
 }
 
 #[derive(Subdiagnostic)]
diff --git a/compiler/rustc_mir_build/src/lib.rs b/compiler/rustc_mir_build/src/lib.rs
index e10a264d3..3f9236c9d 100644
--- a/compiler/rustc_mir_build/src/lib.rs
+++ b/compiler/rustc_mir_build/src/lib.rs
@@ -8,7 +8,6 @@
 #![feature(if_let_guard)]
 #![feature(let_chains)]
 #![feature(min_specialization)]
-#![feature(once_cell)]
 #![feature(try_blocks)]
 #![recursion_limit = "256"]
 
@@ -28,7 +27,7 @@ use rustc_middle::ty::query::Providers;
 use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_macros::fluent_messages;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     providers.check_match = thir::pattern::check_match;
diff --git a/compiler/rustc_mir_build/src/lints.rs b/compiler/rustc_mir_build/src/lints.rs
index f67f24b43..8e41957af 100644
--- a/compiler/rustc_mir_build/src/lints.rs
+++ b/compiler/rustc_mir_build/src/lints.rs
@@ -3,7 +3,7 @@ use rustc_data_structures::graph::iterate::{
     NodeStatus, TriColorDepthFirstSearch, TriColorVisitor,
 };
 use rustc_hir::def::DefKind;
-use rustc_middle::mir::{BasicBlock, BasicBlocks, Body, Operand, TerminatorKind};
+use rustc_middle::mir::{self, BasicBlock, BasicBlocks, Body, Operand, TerminatorKind};
 use rustc_middle::ty::subst::{GenericArg, InternalSubsts};
 use rustc_middle::ty::{self, Instance, TyCtxt};
 use rustc_session::lint::builtin::UNCONDITIONAL_RECURSION;
@@ -18,7 +18,7 @@ pub(crate) fn check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
         let trait_substs = match tcx.trait_of_item(def_id.to_def_id()) {
             Some(trait_def_id) => {
                 let trait_substs_count = tcx.generics_of(trait_def_id).count();
-                &InternalSubsts::identity_for_item(tcx, def_id.to_def_id())[..trait_substs_count]
+                &InternalSubsts::identity_for_item(tcx, def_id)[..trait_substs_count]
             }
             _ => &[],
         };
@@ -108,7 +108,7 @@ impl<'mir, 'tcx> TriColorVisitor<BasicBlocks<'tcx>> for Search<'mir, 'tcx> {
 
         match self.body[bb].terminator().kind {
             // These terminators return control flow to the caller.
-            TerminatorKind::Abort
+            TerminatorKind::Terminate
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::Resume
             | TerminatorKind::Return
@@ -128,7 +128,6 @@ impl<'mir, 'tcx> TriColorVisitor<BasicBlocks<'tcx>> for Search<'mir, 'tcx> {
             TerminatorKind::Assert { .. }
             | TerminatorKind::Call { .. }
             | TerminatorKind::Drop { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::FalseEdge { .. }
             | TerminatorKind::FalseUnwind { .. }
             | TerminatorKind::Goto { .. }
@@ -150,7 +149,9 @@ impl<'mir, 'tcx> TriColorVisitor<BasicBlocks<'tcx>> for Search<'mir, 'tcx> {
 
     fn ignore_edge(&mut self, bb: BasicBlock, target: BasicBlock) -> bool {
         let terminator = self.body[bb].terminator();
-        if terminator.unwind() == Some(&Some(target)) && terminator.successors().count() > 1 {
+        if terminator.unwind() == Some(&mir::UnwindAction::Cleanup(target))
+            && terminator.successors().count() > 1
+        {
             return true;
         }
         // Don't traverse successors of recursive calls or false CFG edges.
diff --git a/compiler/rustc_mir_build/src/thir/cx/block.rs b/compiler/rustc_mir_build/src/thir/cx/block.rs
index 321353ca2..8aacec53f 100644
--- a/compiler/rustc_mir_build/src/thir/cx/block.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/block.rs
@@ -105,6 +105,10 @@ impl<'tcx> Cx<'tcx> {
                             }
                         }
 
+                        let span = match local.init {
+                            Some(init) => local.span.with_hi(init.span.hi()),
+                            None => local.span,
+                        };
                         let stmt = Stmt {
                             kind: StmtKind::Let {
                                 remainder_scope,
@@ -116,6 +120,7 @@ impl<'tcx> Cx<'tcx> {
                                 initializer: local.init.map(|init| self.mirror_expr(init)),
                                 else_block,
                                 lint_level: LintLevel::Explicit(local.hir_id),
+                                span,
                             },
                             opt_destruction_scope: opt_dxn_ext,
                         };
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 9086412c0..8e2e92e6f 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -10,7 +10,7 @@ use rustc_middle::hir::place::Place as HirPlace;
 use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
 use rustc_middle::hir::place::ProjectionKind as HirProjectionKind;
 use rustc_middle::middle::region;
-use rustc_middle::mir::{self, BinOp, BorrowKind, Field, UnOp};
+use rustc_middle::mir::{self, BinOp, BorrowKind, UnOp};
 use rustc_middle::thir::*;
 use rustc_middle::ty::adjustment::{
     Adjust, Adjustment, AutoBorrow, AutoBorrowMutability, PointerCast,
@@ -20,7 +20,7 @@ use rustc_middle::ty::{
     self, AdtKind, InlineConstSubsts, InlineConstSubstsParts, ScalarInt, Ty, UpvarSubsts, UserType,
 };
 use rustc_span::{sym, Span};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, FIRST_VARIANT};
 
 impl<'tcx> Cx<'tcx> {
     pub(crate) fn mirror_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) -> ExprId {
@@ -185,7 +185,7 @@ impl<'tcx> Cx<'tcx> {
         if self.typeck_results().is_coercion_cast(source.hir_id) {
             // Convert the lexpr to a vexpr.
             ExprKind::Use { source: self.mirror_expr(source) }
-        } else if self.typeck_results().expr_ty(source).is_region_ptr() {
+        } else if self.typeck_results().expr_ty(source).is_ref() {
             // Special cased so that we can type check that the element
             // type of the source matches the pointed to type of the
             // destination.
@@ -357,7 +357,7 @@ impl<'tcx> Cx<'tcx> {
                                 Res::Def(DefKind::Ctor(_, CtorKind::Fn), ctor_id) => {
                                     Some((adt_def, adt_def.variant_index_with_ctor_id(ctor_id)))
                                 }
-                                Res::SelfCtor(..) => Some((adt_def, VariantIdx::new(0))),
+                                Res::SelfCtor(..) => Some((adt_def, FIRST_VARIANT)),
                                 _ => None,
                             })
                         } else {
@@ -379,7 +379,7 @@ impl<'tcx> Cx<'tcx> {
                             .iter()
                             .enumerate()
                             .map(|(idx, e)| FieldExpr {
-                                name: Field::new(idx),
+                                name: FieldIdx::new(idx),
                                 expr: self.mirror_expr(e),
                             })
                             .collect();
@@ -510,7 +510,7 @@ impl<'tcx> Cx<'tcx> {
                         debug!("make_mirror_unadjusted: (struct/union) user_ty={:?}", user_ty);
                         ExprKind::Adt(Box::new(AdtExpr {
                             adt_def: *adt,
-                            variant_index: VariantIdx::new(0),
+                            variant_index: FIRST_VARIANT,
                             substs,
                             user_ty,
                             fields: self.field_refs(fields),
@@ -732,8 +732,8 @@ impl<'tcx> Cx<'tcx> {
             }
             hir::ExprKind::Field(ref source, ..) => ExprKind::Field {
                 lhs: self.mirror_expr(source),
-                variant_index: VariantIdx::new(0),
-                name: Field::new(self.typeck_results.field_index(expr.hir_id)),
+                variant_index: FIRST_VARIANT,
+                name: self.typeck_results.field_index(expr.hir_id),
             },
             hir::ExprKind::Cast(ref source, ref cast_ty) => {
                 // Check for a user-given type annotation on this `cast`
@@ -780,7 +780,6 @@ impl<'tcx> Cx<'tcx> {
             hir::ExprKind::DropTemps(ref source) => {
                 ExprKind::Use { source: self.mirror_expr(source) }
             }
-            hir::ExprKind::Box(ref value) => ExprKind::Box { value: self.mirror_expr(value) },
             hir::ExprKind::Array(ref fields) => {
                 ExprKind::Array { fields: self.mirror_exprs(fields) }
             }
@@ -1054,7 +1053,7 @@ impl<'tcx> Cx<'tcx> {
                 HirProjectionKind::Field(field, variant_index) => ExprKind::Field {
                     lhs: self.thir.exprs.push(captured_place_expr),
                     variant_index,
-                    name: Field::new(field as usize),
+                    name: field,
                 },
                 HirProjectionKind::Index | HirProjectionKind::Subslice => {
                     // We don't capture these projections, so we can ignore them here
@@ -1108,7 +1107,7 @@ impl<'tcx> Cx<'tcx> {
         fields
             .iter()
             .map(|field| FieldExpr {
-                name: Field::new(self.typeck_results.field_index(field.hir_id)),
+                name: self.typeck_results.field_index(field.hir_id),
                 expr: self.mirror_expr(field.expr),
             })
             .collect()
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 2640ca56b..8f58db504 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -2,45 +2,48 @@ use super::deconstruct_pat::{Constructor, DeconstructedPat};
 use super::usefulness::{
     compute_match_usefulness, MatchArm, MatchCheckCtxt, Reachability, UsefulnessReport,
 };
-use super::{PatCtxt, PatternError};
 
 use crate::errors::*;
 
-use hir::{ExprKind, PatKind};
 use rustc_arena::TypedArena;
-use rustc_ast::{LitKind, Mutability};
+use rustc_ast::Mutability;
+use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{
     struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, MultiSpan,
 };
 use rustc_hir as hir;
 use rustc_hir::def::*;
-use rustc_hir::def_id::DefId;
-use rustc_hir::intravisit::{self, Visitor};
-use rustc_hir::{HirId, Pat};
+use rustc_hir::def_id::LocalDefId;
+use rustc_hir::HirId;
+use rustc_middle::thir::visit::{self, Visitor};
+use rustc_middle::thir::*;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt};
-
 use rustc_session::lint::builtin::{
     BINDINGS_WITH_VARIANT_NAME, IRREFUTABLE_LET_PATTERNS, UNREACHABLE_PATTERNS,
 };
 use rustc_session::Session;
-use rustc_span::source_map::Spanned;
-use rustc_span::{BytePos, Span};
-
-pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: DefId) {
-    let body_id = match def_id.as_local() {
-        None => return,
-        Some(def_id) => tcx.hir().body_owned_by(def_id),
-    };
+use rustc_span::hygiene::DesugaringKind;
+use rustc_span::Span;
 
+pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: LocalDefId) {
+    let Ok((thir, expr)) = tcx.thir_body(ty::WithOptConstParam::unknown(def_id)) else { return };
+    let thir = thir.borrow();
     let pattern_arena = TypedArena::default();
     let mut visitor = MatchVisitor {
         tcx,
-        typeck_results: tcx.typeck_body(body_id),
+        thir: &*thir,
         param_env: tcx.param_env(def_id),
+        lint_level: tcx.hir().local_def_id_to_hir_id(def_id),
+        let_source: LetSource::None,
         pattern_arena: &pattern_arena,
     };
-    visitor.visit_body(tcx.hir().body(body_id));
+    visitor.visit_expr(&thir[expr]);
+    for param in thir.params.iter() {
+        if let Some(box ref pattern) = param.pat {
+            visitor.check_irrefutable(pattern, "function argument", None);
+        }
+    }
 }
 
 fn create_e0004(
@@ -58,77 +61,132 @@ enum RefutableFlag {
 }
 use RefutableFlag::*;
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum LetSource {
+    None,
+    IfLet,
+    IfLetGuard,
+    LetElse,
+    WhileLet,
+}
+
 struct MatchVisitor<'a, 'p, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    typeck_results: &'a ty::TypeckResults<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
+    thir: &'a Thir<'tcx>,
+    lint_level: HirId,
+    let_source: LetSource,
     pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
 }
 
-impl<'tcx> Visitor<'tcx> for MatchVisitor<'_, '_, 'tcx> {
-    fn visit_expr(&mut self, ex: &'tcx hir::Expr<'tcx>) {
-        intravisit::walk_expr(self, ex);
-        match &ex.kind {
-            hir::ExprKind::Match(scrut, arms, source) => {
-                self.check_match(scrut, arms, *source, ex.span)
+impl<'a, 'tcx> Visitor<'a, 'tcx> for MatchVisitor<'a, '_, 'tcx> {
+    fn thir(&self) -> &'a Thir<'tcx> {
+        self.thir
+    }
+
+    #[instrument(level = "trace", skip(self))]
+    fn visit_arm(&mut self, arm: &Arm<'tcx>) {
+        match arm.guard {
+            Some(Guard::If(expr)) => {
+                self.with_let_source(LetSource::IfLetGuard, |this| {
+                    this.visit_expr(&this.thir[expr])
+                });
             }
-            hir::ExprKind::Let(hir::Let { pat, init, span, .. }) => {
-                self.check_let(pat, init, *span)
+            Some(Guard::IfLet(ref pat, expr)) => {
+                self.with_let_source(LetSource::IfLetGuard, |this| {
+                    this.check_let(pat, expr, LetSource::IfLetGuard, pat.span);
+                    this.visit_pat(pat);
+                    this.visit_expr(&this.thir[expr]);
+                });
             }
-            _ => {}
+            None => {}
         }
+        self.visit_pat(&arm.pattern);
+        self.visit_expr(&self.thir[arm.body]);
     }
 
-    fn visit_local(&mut self, loc: &'tcx hir::Local<'tcx>) {
-        intravisit::walk_local(self, loc);
-        let els = loc.els;
-        if let Some(init) = loc.init && els.is_some() {
-            // Build a span without the else { ... } as we don't want to underline
-            // the entire else block in the IDE setting.
-            let span = loc.span.with_hi(init.span.hi());
-            self.check_let(&loc.pat, init, span);
-        }
-
-        let (msg, sp) = match loc.source {
-            hir::LocalSource::Normal => ("local binding", Some(loc.span)),
-            hir::LocalSource::AsyncFn => ("async fn binding", None),
-            hir::LocalSource::AwaitDesugar => ("`await` future binding", None),
-            hir::LocalSource::AssignDesugar(_) => ("destructuring assignment binding", None),
+    #[instrument(level = "trace", skip(self))]
+    fn visit_expr(&mut self, ex: &Expr<'tcx>) {
+        match ex.kind {
+            ExprKind::Scope { value, lint_level, .. } => {
+                let old_lint_level = self.lint_level;
+                if let LintLevel::Explicit(hir_id) = lint_level {
+                    self.lint_level = hir_id;
+                }
+                self.visit_expr(&self.thir[value]);
+                self.lint_level = old_lint_level;
+                return;
+            }
+            ExprKind::If { cond, then, else_opt, if_then_scope: _ } => {
+                // Give a specific `let_source` for the condition.
+                let let_source = match ex.span.desugaring_kind() {
+                    Some(DesugaringKind::WhileLoop) => LetSource::WhileLet,
+                    _ => LetSource::IfLet,
+                };
+                self.with_let_source(let_source, |this| this.visit_expr(&self.thir[cond]));
+                self.with_let_source(LetSource::None, |this| {
+                    this.visit_expr(&this.thir[then]);
+                    if let Some(else_) = else_opt {
+                        this.visit_expr(&this.thir[else_]);
+                    }
+                });
+                return;
+            }
+            ExprKind::Match { scrutinee, box ref arms } => {
+                let source = match ex.span.desugaring_kind() {
+                    Some(DesugaringKind::ForLoop) => hir::MatchSource::ForLoopDesugar,
+                    Some(DesugaringKind::QuestionMark) => hir::MatchSource::TryDesugar,
+                    Some(DesugaringKind::Await) => hir::MatchSource::AwaitDesugar,
+                    _ => hir::MatchSource::Normal,
+                };
+                self.check_match(scrutinee, arms, source, ex.span);
+            }
+            ExprKind::Let { box ref pat, expr } => {
+                self.check_let(pat, expr, self.let_source, ex.span);
+            }
+            ExprKind::LogicalOp { op: LogicalOp::And, lhs, rhs } => {
+                self.check_let_chain(self.let_source, ex.span, lhs, rhs);
+            }
+            _ => {}
         };
-        if els.is_none() {
-            self.check_irrefutable(&loc.pat, msg, sp);
-        }
+        self.with_let_source(LetSource::None, |this| visit::walk_expr(this, ex));
     }
 
-    fn visit_param(&mut self, param: &'tcx hir::Param<'tcx>) {
-        intravisit::walk_param(self, param);
-        self.check_irrefutable(&param.pat, "function argument", None);
-    }
-}
-
-impl PatCtxt<'_, '_> {
-    fn report_inlining_errors(&self) {
-        for error in &self.errors {
-            match *error {
-                PatternError::StaticInPattern(span) => {
-                    self.tcx.sess.emit_err(StaticInPattern { span });
+    fn visit_stmt(&mut self, stmt: &Stmt<'tcx>) {
+        let old_lint_level = self.lint_level;
+        match stmt.kind {
+            StmtKind::Let {
+                box ref pattern, initializer, else_block, lint_level, span, ..
+            } => {
+                if let LintLevel::Explicit(lint_level) = lint_level {
+                    self.lint_level = lint_level;
                 }
-                PatternError::AssocConstInPattern(span) => {
-                    self.tcx.sess.emit_err(AssocConstInPattern { span });
-                }
-                PatternError::ConstParamInPattern(span) => {
-                    self.tcx.sess.emit_err(ConstParamInPattern { span });
+
+                if let Some(initializer) = initializer && else_block.is_some() {
+                    self.check_let(pattern, initializer, LetSource::LetElse, span);
                 }
-                PatternError::NonConstPath(span) => {
-                    self.tcx.sess.emit_err(NonConstPath { span });
+
+                if else_block.is_none() {
+                    self.check_irrefutable(pattern, "local binding", Some(span));
                 }
             }
+            _ => {}
         }
+        visit::walk_stmt(self, stmt);
+        self.lint_level = old_lint_level;
     }
 }
 
 impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
-    fn check_patterns(&self, pat: &Pat<'_>, rf: RefutableFlag) {
+    #[instrument(level = "trace", skip(self, f))]
+    fn with_let_source(&mut self, let_source: LetSource, f: impl FnOnce(&mut Self)) {
+        let old_let_source = self.let_source;
+        self.let_source = let_source;
+        ensure_sufficient_stack(|| f(self));
+        self.let_source = old_let_source;
+    }
+
+    fn check_patterns(&self, pat: &Pat<'tcx>, rf: RefutableFlag) {
         pat.walk_always(|pat| check_borrow_conflicts_in_at_patterns(self, pat));
         check_for_bindings_named_same_as_variants(self, pat, rf);
     }
@@ -136,73 +194,63 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
     fn lower_pattern(
         &self,
         cx: &mut MatchCheckCtxt<'p, 'tcx>,
-        pat: &'tcx hir::Pat<'tcx>,
-        have_errors: &mut bool,
+        pattern: &Pat<'tcx>,
     ) -> &'p DeconstructedPat<'p, 'tcx> {
-        let mut patcx = PatCtxt::new(self.tcx, self.param_env, self.typeck_results);
-        patcx.include_lint_checks();
-        let pattern = patcx.lower_pattern(pat);
-        let pattern: &_ = cx.pattern_arena.alloc(DeconstructedPat::from_pat(cx, &pattern));
-        if !patcx.errors.is_empty() {
-            *have_errors = true;
-            patcx.report_inlining_errors();
-        }
-        pattern
+        cx.pattern_arena.alloc(DeconstructedPat::from_pat(cx, &pattern))
     }
 
-    fn new_cx(&self, hir_id: HirId) -> MatchCheckCtxt<'p, 'tcx> {
+    fn new_cx(&self, hir_id: HirId, refutable: bool) -> MatchCheckCtxt<'p, 'tcx> {
         MatchCheckCtxt {
             tcx: self.tcx,
             param_env: self.param_env,
             module: self.tcx.parent_module(hir_id).to_def_id(),
             pattern_arena: &self.pattern_arena,
+            refutable,
         }
     }
 
-    fn check_let(&mut self, pat: &'tcx hir::Pat<'tcx>, scrutinee: &hir::Expr<'_>, span: Span) {
+    #[instrument(level = "trace", skip(self))]
+    fn check_let(&mut self, pat: &Pat<'tcx>, scrutinee: ExprId, source: LetSource, span: Span) {
+        if let LetSource::None = source {
+            return;
+        }
         self.check_patterns(pat, Refutable);
-        let mut cx = self.new_cx(scrutinee.hir_id);
-        let tpat = self.lower_pattern(&mut cx, pat, &mut false);
-        self.check_let_reachability(&mut cx, pat.hir_id, tpat, span);
+        let mut cx = self.new_cx(self.lint_level, true);
+        let tpat = self.lower_pattern(&mut cx, pat);
+        self.check_let_reachability(&mut cx, self.lint_level, source, tpat, span);
     }
 
     fn check_match(
         &mut self,
-        scrut: &hir::Expr<'_>,
-        hir_arms: &'tcx [hir::Arm<'tcx>],
+        scrut: ExprId,
+        arms: &[ArmId],
         source: hir::MatchSource,
         expr_span: Span,
     ) {
-        let mut cx = self.new_cx(scrut.hir_id);
+        let mut cx = self.new_cx(self.lint_level, true);
 
-        for arm in hir_arms {
+        for &arm in arms {
             // Check the arm for some things unrelated to exhaustiveness.
-            self.check_patterns(&arm.pat, Refutable);
-            if let Some(hir::Guard::IfLet(ref let_expr)) = arm.guard {
-                self.check_patterns(let_expr.pat, Refutable);
-                let tpat = self.lower_pattern(&mut cx, let_expr.pat, &mut false);
-                self.check_let_reachability(&mut cx, let_expr.pat.hir_id, tpat, tpat.span());
-            }
+            let arm = &self.thir.arms[arm];
+            self.check_patterns(&arm.pattern, Refutable);
         }
 
-        let mut have_errors = false;
-
-        let arms: Vec<_> = hir_arms
+        let tarms: Vec<_> = arms
             .iter()
-            .map(|hir::Arm { pat, guard, .. }| MatchArm {
-                pat: self.lower_pattern(&mut cx, pat, &mut have_errors),
-                hir_id: pat.hir_id,
-                has_guard: guard.is_some(),
+            .map(|&arm| {
+                let arm = &self.thir.arms[arm];
+                let hir_id = match arm.lint_level {
+                    LintLevel::Explicit(hir_id) => hir_id,
+                    LintLevel::Inherited => self.lint_level,
+                };
+                let pat = self.lower_pattern(&mut cx, &arm.pattern);
+                MatchArm { pat, hir_id, has_guard: arm.guard.is_some() }
             })
             .collect();
 
-        // Bail out early if lowering failed.
-        if have_errors {
-            return;
-        }
-
-        let scrut_ty = self.typeck_results.expr_ty_adjusted(scrut);
-        let report = compute_match_usefulness(&cx, &arms, scrut.hir_id, scrut_ty);
+        let scrut = &self.thir[scrut];
+        let scrut_ty = scrut.ty;
+        let report = compute_match_usefulness(&cx, &tarms, self.lint_level, scrut_ty);
 
         match source {
             // Don't report arm reachability of desugared `match $iter.into_iter() { iter => .. }`
@@ -219,12 +267,18 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
         // Check if the match is exhaustive.
         let witnesses = report.non_exhaustiveness_witnesses;
         if !witnesses.is_empty() {
-            if source == hir::MatchSource::ForLoopDesugar && hir_arms.len() == 2 {
+            if source == hir::MatchSource::ForLoopDesugar && arms.len() == 2 {
                 // the for loop pattern is not irrefutable
-                let pat = hir_arms[1].pat.for_loop_some().unwrap();
-                self.check_irrefutable(pat, "`for` loop binding", None);
+                let pat = &self.thir[arms[1]].pattern;
+                // `pat` should be `Some(<pat_field>)` from a desugared for loop.
+                debug_assert_eq!(pat.span.desugaring_kind(), Some(DesugaringKind::ForLoop));
+                let PatKind::Variant { ref subpatterns, .. } = pat.kind else { bug!() };
+                let [pat_field] = &subpatterns[..] else { bug!() };
+                self.check_irrefutable(&pat_field.pattern, "`for` loop binding", None);
             } else {
-                non_exhaustive_match(&cx, scrut_ty, scrut.span, witnesses, hir_arms, expr_span);
+                non_exhaustive_match(
+                    &cx, self.thir, scrut_ty, scrut.span, witnesses, arms, expr_span,
+                );
             }
         }
     }
@@ -233,114 +287,93 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
         &mut self,
         cx: &mut MatchCheckCtxt<'p, 'tcx>,
         pat_id: HirId,
+        source: LetSource,
         pat: &'p DeconstructedPat<'p, 'tcx>,
         span: Span,
     ) {
-        if self.check_let_chain(cx, pat_id) {
-            return;
-        }
-
         if is_let_irrefutable(cx, pat_id, pat) {
-            irrefutable_let_pattern(cx.tcx, pat_id, span);
+            irrefutable_let_patterns(cx.tcx, pat_id, source, 1, span);
         }
     }
 
-    fn check_let_chain(&mut self, cx: &mut MatchCheckCtxt<'p, 'tcx>, pat_id: HirId) -> bool {
-        let hir = self.tcx.hir();
-        let parent = hir.parent_id(pat_id);
+    #[instrument(level = "trace", skip(self))]
+    fn check_let_chain(
+        &mut self,
+        let_source: LetSource,
+        top_expr_span: Span,
+        mut lhs: ExprId,
+        rhs: ExprId,
+    ) {
+        if let LetSource::None = let_source {
+            return;
+        }
 
-        // First, figure out if the given pattern is part of a let chain,
-        // and if so, obtain the top node of the chain.
-        let mut top = parent;
-        let mut part_of_chain = false;
-        loop {
-            let new_top = hir.parent_id(top);
-            if let hir::Node::Expr(
-                hir::Expr {
-                    kind: hir::ExprKind::Binary(Spanned { node: hir::BinOpKind::And, .. }, lhs, rhs),
-                    ..
-                },
-                ..,
-            ) = hir.get(new_top)
-            {
-                // If this isn't the first iteration, we need to check
-                // if there is a let expr before us in the chain, so
-                // that we avoid doubly checking the let chain.
-
-                // The way a chain of &&s is encoded is ((let ... && let ...) && let ...) && let ...
-                // as && is left-to-right associative. Thus, we need to check rhs.
-                if part_of_chain && matches!(rhs.kind, hir::ExprKind::Let(..)) {
-                    return true;
+        // Lint level enclosing the next `lhs`.
+        let mut cur_lint_level = self.lint_level;
+
+        // Obtain the refutabilities of all exprs in the chain,
+        // and record chain members that aren't let exprs.
+        let mut chain_refutabilities = Vec::new();
+
+        let add = |expr: ExprId, mut local_lint_level| {
+            // `local_lint_level` is the lint level enclosing the pattern inside `expr`.
+            let mut expr = &self.thir[expr];
+            debug!(?expr, ?local_lint_level, "add");
+            // Fast-forward through scopes.
+            while let ExprKind::Scope { value, lint_level, .. } = expr.kind {
+                if let LintLevel::Explicit(hir_id) = lint_level {
+                    local_lint_level = hir_id
                 }
-                // If there is a let at the lhs, and we provide the rhs, we don't do any checking either.
-                if !part_of_chain && matches!(lhs.kind, hir::ExprKind::Let(..)) && rhs.hir_id == top
-                {
-                    return true;
+                expr = &self.thir[value];
+            }
+            debug!(?expr, ?local_lint_level, "after scopes");
+            match expr.kind {
+                ExprKind::Let { box ref pat, expr: _ } => {
+                    let mut ncx = self.new_cx(local_lint_level, true);
+                    let tpat = self.lower_pattern(&mut ncx, pat);
+                    let refutable = !is_let_irrefutable(&mut ncx, local_lint_level, tpat);
+                    Some((expr.span, refutable))
                 }
-            } else {
-                // We've reached the top.
-                break;
+                _ => None,
             }
+        };
 
-            // Since this function is called within a let context, it is reasonable to assume that any parent
-            // `&&` infers a let chain
-            part_of_chain = true;
-            top = new_top;
-        }
-        if !part_of_chain {
-            return false;
-        }
+        // Let chains recurse on the left, so we start by adding the rightmost.
+        chain_refutabilities.push(add(rhs, cur_lint_level));
 
-        // Second, obtain the refutabilities of all exprs in the chain,
-        // and record chain members that aren't let exprs.
-        let mut chain_refutabilities = Vec::new();
-        let hir::Node::Expr(top_expr) = hir.get(top) else {
-            // We ensure right above that it's an Expr
-            unreachable!()
-        };
-        let mut cur_expr = top_expr;
         loop {
-            let mut add = |expr: &hir::Expr<'tcx>| {
-                let refutability = match expr.kind {
-                    hir::ExprKind::Let(hir::Let { pat, init, span, .. }) => {
-                        let mut ncx = self.new_cx(init.hir_id);
-                        let tpat = self.lower_pattern(&mut ncx, pat, &mut false);
-
-                        let refutable = !is_let_irrefutable(&mut ncx, pat.hir_id, tpat);
-                        Some((*span, refutable))
-                    }
-                    _ => None,
-                };
-                chain_refutabilities.push(refutability);
-            };
-            if let hir::Expr {
-                kind: hir::ExprKind::Binary(Spanned { node: hir::BinOpKind::And, .. }, lhs, rhs),
-                ..
-            } = cur_expr
+            while let ExprKind::Scope { value, lint_level, .. } = self.thir[lhs].kind {
+                if let LintLevel::Explicit(hir_id) = lint_level {
+                    cur_lint_level = hir_id
+                }
+                lhs = value;
+            }
+            if let ExprKind::LogicalOp { op: LogicalOp::And, lhs: new_lhs, rhs: expr } =
+                self.thir[lhs].kind
             {
-                add(rhs);
-                cur_expr = lhs;
+                chain_refutabilities.push(add(expr, cur_lint_level));
+                lhs = new_lhs;
             } else {
-                add(cur_expr);
+                chain_refutabilities.push(add(lhs, cur_lint_level));
                 break;
             }
         }
+        debug!(?chain_refutabilities);
         chain_refutabilities.reverse();
 
         // Third, emit the actual warnings.
-
         if chain_refutabilities.iter().all(|r| matches!(*r, Some((_, false)))) {
             // The entire chain is made up of irrefutable `let` statements
-            let let_source = let_source_parent(self.tcx, top, None);
             irrefutable_let_patterns(
-                cx.tcx,
-                top,
+                self.tcx,
+                self.lint_level,
                 let_source,
                 chain_refutabilities.len(),
-                top_expr.span,
+                top_expr_span,
             );
-            return true;
+            return;
         }
+
         if let Some(until) = chain_refutabilities.iter().position(|r| !matches!(*r, Some((_, false)))) && until > 0 {
             // The chain has a non-zero prefix of irrefutable `let` statements.
 
@@ -350,7 +383,6 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
             // so can't always be moved out.
             // FIXME: Add checking whether the bindings are actually used in the prefix,
             // and lint if they are not.
-            let let_source = let_source_parent(self.tcx, top, None);
             if !matches!(let_source, LetSource::WhileLet | LetSource::IfLetGuard) {
                 // Emit the lint
                 let prefix = &chain_refutabilities[..until];
@@ -358,9 +390,10 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
                 let span_end = prefix.last().unwrap().unwrap().0;
                 let span = span_start.to(span_end);
                 let count = prefix.len();
-                cx.tcx.emit_spanned_lint(IRREFUTABLE_LET_PATTERNS, top, span, LeadingIrrefutableLetPatterns { count });
+                self.tcx.emit_spanned_lint(IRREFUTABLE_LET_PATTERNS, self.lint_level, span, LeadingIrrefutableLetPatterns { count });
             }
         }
+
         if let Some(from) = chain_refutabilities.iter().rposition(|r| !matches!(*r, Some((_, false)))) && from != (chain_refutabilities.len() - 1) {
             // The chain has a non-empty suffix of irrefutable `let` statements
             let suffix = &chain_refutabilities[from + 1..];
@@ -368,18 +401,18 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
             let span_end = suffix.last().unwrap().unwrap().0;
             let span = span_start.to(span_end);
             let count = suffix.len();
-            cx.tcx.emit_spanned_lint(IRREFUTABLE_LET_PATTERNS, top, span, TrailingIrrefutableLetPatterns { count });
+            self.tcx.emit_spanned_lint(IRREFUTABLE_LET_PATTERNS, self.lint_level, span, TrailingIrrefutableLetPatterns { count });
         }
-        true
     }
 
-    fn check_irrefutable(&self, pat: &'tcx Pat<'tcx>, origin: &str, sp: Option<Span>) {
-        let mut cx = self.new_cx(pat.hir_id);
+    #[instrument(level = "trace", skip(self))]
+    fn check_irrefutable(&self, pat: &Pat<'tcx>, origin: &str, sp: Option<Span>) {
+        let mut cx = self.new_cx(self.lint_level, false);
 
-        let pattern = self.lower_pattern(&mut cx, pat, &mut false);
+        let pattern = self.lower_pattern(&mut cx, pat);
         let pattern_ty = pattern.ty();
-        let arm = MatchArm { pat: pattern, hir_id: pat.hir_id, has_guard: false };
-        let report = compute_match_usefulness(&cx, &[arm], pat.hir_id, pattern_ty);
+        let arm = MatchArm { pat: pattern, hir_id: self.lint_level, has_guard: false };
+        let report = compute_match_usefulness(&cx, &[arm], self.lint_level, pattern_ty);
 
         // Note: we ignore whether the pattern is unreachable (i.e. whether the type is empty). We
         // only care about exhaustiveness here.
@@ -390,58 +423,45 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
             return;
         }
 
-        let (inform, interpreted_as_const, res_defined_here,let_suggestion, misc_suggestion) =
-            if let hir::PatKind::Path(hir::QPath::Resolved(
-                None,
-                hir::Path {
-                    segments: &[hir::PathSegment { args: None, res, ident, .. }],
-                    ..
-                },
-            )) = &pat.kind
-            {
-                (
-                    None,
-                    Some(InterpretedAsConst {
-                        span: pat.span,
-                        article: res.article(),
-                        variable: ident.to_string().to_lowercase(),
-                        res,
-                    }),
-                    try {
-                        ResDefinedHere {
-                            def_span: cx.tcx.hir().res_span(res)?,
-                            res,
-                        }
-                    },
-                    None,
-                    None,
-                )
-            } else if let Some(span) = sp && self.tcx.sess.source_map().is_span_accessible(span) {
-                let mut bindings = vec![];
-                pat.walk_always(&mut |pat: &hir::Pat<'_>| {
-                    if let hir::PatKind::Binding(_, _, ident, _) = pat.kind {
-                        bindings.push(ident);
-                    }
+        let inform = sp.is_some().then_some(Inform);
+        let mut let_suggestion = None;
+        let mut misc_suggestion = None;
+        let mut interpreted_as_const = None;
+        if let PatKind::Constant { .. } = pat.kind
+            && let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(pat.span)
+        {
+            // If the pattern to match is an integer literal:
+            if snippet.chars().all(|c| c.is_digit(10)) {
+                // Then give a suggestion, the user might've meant to create a binding instead.
+                misc_suggestion = Some(MiscPatternSuggestion::AttemptedIntegerLiteral {
+                    start_span: pat.span.shrink_to_lo()
                 });
-                let semi_span = span.shrink_to_hi().with_lo(span.hi() - BytePos(1));
-                let start_span = span.shrink_to_lo();
-                let end_span = semi_span.shrink_to_lo();
-                let count = witnesses.len();
-
-                // If the pattern to match is an integer literal:
-                let int_suggestion = if
-                    let PatKind::Lit(expr) = &pat.kind
-                    && bindings.is_empty()
-                    && let ExprKind::Lit(Spanned { node: LitKind::Int(_, _), span }) = expr.kind {
-                    // Then give a suggestion, the user might've meant to create a binding instead.
-                    Some(MiscPatternSuggestion::AttemptedIntegerLiteral { start_span: span.shrink_to_lo() })
-                } else { None };
-
-                let let_suggestion = if bindings.is_empty() {SuggestLet::If{start_span, semi_span, count}} else{ SuggestLet::Else{end_span, count }};
-                (sp.map(|_|Inform), None, None, Some(let_suggestion), int_suggestion)
-            } else{
-                (sp.map(|_|Inform), None, None,  None, None)
-            };
+            } else if snippet.chars().all(|c| c.is_alphanumeric() || c == '_') {
+                interpreted_as_const = Some(InterpretedAsConst {
+                    span: pat.span,
+                    variable: snippet,
+                });
+            }
+        }
+
+        if let Some(span) = sp
+            && self.tcx.sess.source_map().is_span_accessible(span)
+            && interpreted_as_const.is_none()
+        {
+            let mut bindings = vec![];
+            pat.each_binding(|name, _, _, _| bindings.push(name));
+
+            let semi_span = span.shrink_to_hi();
+            let start_span = span.shrink_to_lo();
+            let end_span = semi_span.shrink_to_lo();
+            let count = witnesses.len();
+
+            let_suggestion = Some(if bindings.is_empty() {
+                SuggestLet::If { start_span, semi_span, count }
+            } else {
+                SuggestLet::Else { end_span, count }
+            });
+        };
 
         let adt_defined_here = try {
             let ty = pattern_ty.peel_refs();
@@ -465,7 +485,6 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
             pattern_ty,
             let_suggestion,
             misc_suggestion,
-            res_defined_here,
             adt_defined_here,
         });
     }
@@ -477,14 +496,18 @@ fn check_for_bindings_named_same_as_variants(
     rf: RefutableFlag,
 ) {
     pat.walk_always(|p| {
-        if let hir::PatKind::Binding(_, _, ident, None) = p.kind
-            && let Some(ty::BindByValue(hir::Mutability::Not)) =
-                cx.typeck_results.extract_binding_mode(cx.tcx.sess, p.hir_id, p.span)
-            && let pat_ty = cx.typeck_results.pat_ty(p).peel_refs()
-            && let ty::Adt(edef, _) = pat_ty.kind()
+        if let PatKind::Binding {
+                name,
+                mode: BindingMode::ByValue,
+                mutability: Mutability::Not,
+                subpattern: None,
+                ty,
+                ..
+            } = p.kind
+            && let ty::Adt(edef, _) = ty.peel_refs().kind()
             && edef.is_enum()
             && edef.variants().iter().any(|variant| {
-                variant.ident(cx.tcx) == ident && variant.ctor_kind() == Some(CtorKind::Const)
+                variant.name == name && variant.ctor_kind() == Some(CtorKind::Const)
             })
         {
             let variant_count = edef.variants().len();
@@ -493,7 +516,7 @@ fn check_for_bindings_named_same_as_variants(
             });
             cx.tcx.emit_spanned_lint(
                 BINDINGS_WITH_VARIANT_NAME,
-                p.hir_id,
+                cx.lint_level,
                 p.span,
                 BindingsWithVariantName {
                     // If this is an irrefutable pattern, and there's > 1 variant,
@@ -503,7 +526,7 @@ fn check_for_bindings_named_same_as_variants(
                         Some(p.span)
                     } else { None },
                     ty_path,
-                    ident,
+                    name,
                 },
             )
         }
@@ -529,11 +552,6 @@ fn unreachable_pattern(tcx: TyCtxt<'_>, span: Span, id: HirId, catchall: Option<
     );
 }
 
-fn irrefutable_let_pattern(tcx: TyCtxt<'_>, id: HirId, span: Span) {
-    let source = let_source(tcx, id);
-    irrefutable_let_patterns(tcx, id, source, 1, span);
-}
-
 fn irrefutable_let_patterns(
     tcx: TyCtxt<'_>,
     id: HirId,
@@ -548,7 +566,7 @@ fn irrefutable_let_patterns(
     }
 
     match source {
-        LetSource::GenericLet => emit_diag!(IrrefutableLetPatternsGenericLet),
+        LetSource::None => bug!(),
         LetSource::IfLet => emit_diag!(IrrefutableLetPatternsIfLet),
         LetSource::IfLetGuard => emit_diag!(IrrefutableLetPatternsIfLetGuard),
         LetSource::LetElse => emit_diag!(IrrefutableLetPatternsLetElse),
@@ -604,10 +622,11 @@ fn report_arm_reachability<'p, 'tcx>(
 /// Report that a match is not exhaustive.
 fn non_exhaustive_match<'p, 'tcx>(
     cx: &MatchCheckCtxt<'p, 'tcx>,
+    thir: &Thir<'tcx>,
     scrut_ty: Ty<'tcx>,
     sp: Span,
     witnesses: Vec<DeconstructedPat<'p, 'tcx>>,
-    arms: &[hir::Arm<'tcx>],
+    arms: &[ArmId],
     expr_span: Span,
 ) {
     let is_empty_match = arms.is_empty();
@@ -705,6 +724,7 @@ fn non_exhaustive_match<'p, 'tcx>(
             ));
         }
         [only] => {
+            let only = &thir[*only];
             let (pre_indentation, is_multiline) = if let Some(snippet) = sm.indentation_before(only.span)
                 && let Ok(with_trailing) = sm.span_extend_while(only.span, |c| c.is_whitespace() || c == ',')
                 && sm.is_multiline(with_trailing)
@@ -713,8 +733,9 @@ fn non_exhaustive_match<'p, 'tcx>(
             } else {
                 (" ".to_string(), false)
             };
-            let comma = if matches!(only.body.kind, hir::ExprKind::Block(..))
-                && only.span.eq_ctxt(only.body.span)
+            let only_body = &thir[only.body];
+            let comma = if matches!(only_body.kind, ExprKind::Block { .. })
+                && only.span.eq_ctxt(only_body.span)
                 && is_multiline
             {
                 ""
@@ -726,24 +747,29 @@ fn non_exhaustive_match<'p, 'tcx>(
                 format!("{}{}{} => todo!()", comma, pre_indentation, pattern),
             ));
         }
-        [.., prev, last] if prev.span.eq_ctxt(last.span) => {
-            let comma = if matches!(last.body.kind, hir::ExprKind::Block(..))
-                && last.span.eq_ctxt(last.body.span)
-            {
-                ""
-            } else {
-                ","
-            };
-            let spacing = if sm.is_multiline(prev.span.between(last.span)) {
-                sm.indentation_before(last.span).map(|indent| format!("\n{indent}"))
-            } else {
-                Some(" ".to_string())
-            };
-            if let Some(spacing) = spacing {
-                suggestion = Some((
-                    last.span.shrink_to_hi(),
-                    format!("{}{}{} => todo!()", comma, spacing, pattern),
-                ));
+        [.., prev, last] => {
+            let prev = &thir[*prev];
+            let last = &thir[*last];
+            if prev.span.eq_ctxt(last.span) {
+                let last_body = &thir[last.body];
+                let comma = if matches!(last_body.kind, ExprKind::Block { .. })
+                    && last.span.eq_ctxt(last_body.span)
+                {
+                    ""
+                } else {
+                    ","
+                };
+                let spacing = if sm.is_multiline(prev.span.between(last.span)) {
+                    sm.indentation_before(last.span).map(|indent| format!("\n{indent}"))
+                } else {
+                    Some(" ".to_string())
+                };
+                if let Some(spacing) = spacing {
+                    suggestion = Some((
+                        last.span.shrink_to_hi(),
+                        format!("{}{}{} => todo!()", comma, spacing, pattern),
+                    ));
+                }
             }
         }
         _ => {}
@@ -863,10 +889,6 @@ fn maybe_point_at_variant<'a, 'p: 'a, 'tcx: 'a>(
 }
 
 /// Check if a by-value binding is by-value. That is, check if the binding's type is not `Copy`.
-fn is_binding_by_move(cx: &MatchVisitor<'_, '_, '_>, hir_id: HirId) -> bool {
-    !cx.typeck_results.node_type(hir_id).is_copy_modulo_regions(cx.tcx, cx.param_env)
-}
-
 /// Check that there are no borrow or move conflicts in `binding @ subpat` patterns.
 ///
 /// For example, this would reject:
@@ -877,45 +899,36 @@ fn is_binding_by_move(cx: &MatchVisitor<'_, '_, '_>, hir_id: HirId) -> bool {
 /// - `x @ Some(ref mut? y)`.
 ///
 /// This analysis is *not* subsumed by NLL.
-fn check_borrow_conflicts_in_at_patterns(cx: &MatchVisitor<'_, '_, '_>, pat: &Pat<'_>) {
+fn check_borrow_conflicts_in_at_patterns<'tcx>(cx: &MatchVisitor<'_, '_, 'tcx>, pat: &Pat<'tcx>) {
     // Extract `sub` in `binding @ sub`.
-    let (name, sub) = match &pat.kind {
-        hir::PatKind::Binding(.., name, Some(sub)) => (*name, sub),
-        _ => return,
-    };
-    let binding_span = pat.span.with_hi(name.span.hi());
+    let PatKind::Binding { name, mode, ty, subpattern: Some(box ref sub), .. } = pat.kind else { return };
+
+    let is_binding_by_move = |ty: Ty<'tcx>| !ty.is_copy_modulo_regions(cx.tcx, cx.param_env);
 
-    let typeck_results = cx.typeck_results;
     let sess = cx.tcx.sess;
 
     // Get the binding move, extract the mutability if by-ref.
-    let mut_outer = match typeck_results.extract_binding_mode(sess, pat.hir_id, pat.span) {
-        Some(ty::BindByValue(_)) if is_binding_by_move(cx, pat.hir_id) => {
+    let mut_outer = match mode {
+        BindingMode::ByValue if is_binding_by_move(ty) => {
             // We have `x @ pat` where `x` is by-move. Reject all borrows in `pat`.
             let mut conflicts_ref = Vec::new();
-            sub.each_binding(|_, hir_id, span, _| {
-                match typeck_results.extract_binding_mode(sess, hir_id, span) {
-                    Some(ty::BindByValue(_)) | None => {}
-                    Some(ty::BindByReference(_)) => conflicts_ref.push(span),
-                }
+            sub.each_binding(|_, mode, _, span| match mode {
+                BindingMode::ByValue => {}
+                BindingMode::ByRef(_) => conflicts_ref.push(span),
             });
             if !conflicts_ref.is_empty() {
                 sess.emit_err(BorrowOfMovedValue {
-                    span: pat.span,
-                    binding_span,
+                    binding_span: pat.span,
                     conflicts_ref,
                     name,
-                    ty: typeck_results.node_type(pat.hir_id),
-                    suggest_borrowing: pat
-                        .span
-                        .contains(binding_span)
-                        .then(|| binding_span.shrink_to_lo()),
+                    ty,
+                    suggest_borrowing: Some(pat.span.shrink_to_lo()),
                 });
             }
             return;
         }
-        Some(ty::BindByValue(_)) | None => return,
-        Some(ty::BindByReference(m)) => m,
+        BindingMode::ByValue => return,
+        BindingMode::ByRef(m) => m.mutability(),
     };
 
     // We now have `ref $mut_outer binding @ sub` (semantically).
@@ -923,9 +936,9 @@ fn check_borrow_conflicts_in_at_patterns(cx: &MatchVisitor<'_, '_, '_>, pat: &Pa
     let mut conflicts_move = Vec::new();
     let mut conflicts_mut_mut = Vec::new();
     let mut conflicts_mut_ref = Vec::new();
-    sub.each_binding(|_, hir_id, span, name| {
-        match typeck_results.extract_binding_mode(sess, hir_id, span) {
-            Some(ty::BindByReference(mut_inner)) => match (mut_outer, mut_inner) {
+    sub.each_binding(|name, mode, ty, span| {
+        match mode {
+            BindingMode::ByRef(mut_inner) => match (mut_outer, mut_inner.mutability()) {
                 // Both sides are `ref`.
                 (Mutability::Not, Mutability::Not) => {}
                 // 2x `ref mut`.
@@ -939,10 +952,10 @@ fn check_borrow_conflicts_in_at_patterns(cx: &MatchVisitor<'_, '_, '_>, pat: &Pa
                     conflicts_mut_ref.push(Conflict::Ref { span, name })
                 }
             },
-            Some(ty::BindByValue(_)) if is_binding_by_move(cx, hir_id) => {
+            BindingMode::ByValue if is_binding_by_move(ty) => {
                 conflicts_move.push(Conflict::Moved { span, name }) // `ref mut?` + by-move conflict.
             }
-            Some(ty::BindByValue(_)) | None => {} // `ref mut?` + by-copy is fine.
+            BindingMode::ByValue => {} // `ref mut?` + by-copy is fine.
         }
     });
 
@@ -950,92 +963,30 @@ fn check_borrow_conflicts_in_at_patterns(cx: &MatchVisitor<'_, '_, '_>, pat: &Pa
     let report_mut_ref = !conflicts_mut_ref.is_empty();
     let report_move_conflict = !conflicts_move.is_empty();
 
-    let mut occurences = match mut_outer {
-        Mutability::Mut => vec![Conflict::Mut { span: binding_span, name }],
-        Mutability::Not => vec![Conflict::Ref { span: binding_span, name }],
+    let mut occurrences = match mut_outer {
+        Mutability::Mut => vec![Conflict::Mut { span: pat.span, name }],
+        Mutability::Not => vec![Conflict::Ref { span: pat.span, name }],
     };
-    occurences.extend(conflicts_mut_mut);
-    occurences.extend(conflicts_mut_ref);
-    occurences.extend(conflicts_move);
+    occurrences.extend(conflicts_mut_mut);
+    occurrences.extend(conflicts_mut_ref);
+    occurrences.extend(conflicts_move);
 
     // Report errors if any.
     if report_mut_mut {
         // Report mutability conflicts for e.g. `ref mut x @ Some(ref mut y)`.
-        sess.emit_err(MultipleMutBorrows { span: pat.span, occurences });
+        sess.emit_err(MultipleMutBorrows { span: pat.span, occurrences });
     } else if report_mut_ref {
         // Report mutability conflicts for e.g. `ref x @ Some(ref mut y)` or the converse.
         match mut_outer {
             Mutability::Mut => {
-                sess.emit_err(AlreadyMutBorrowed { span: pat.span, occurences });
+                sess.emit_err(AlreadyMutBorrowed { span: pat.span, occurrences });
             }
             Mutability::Not => {
-                sess.emit_err(AlreadyBorrowed { span: pat.span, occurences });
+                sess.emit_err(AlreadyBorrowed { span: pat.span, occurrences });
             }
         };
     } else if report_move_conflict {
         // Report by-ref and by-move conflicts, e.g. `ref x @ y`.
-        sess.emit_err(MovedWhileBorrowed { span: pat.span, occurences });
-    }
-}
-
-#[derive(Clone, Copy, Debug)]
-pub enum LetSource {
-    GenericLet,
-    IfLet,
-    IfLetGuard,
-    LetElse,
-    WhileLet,
-}
-
-fn let_source(tcx: TyCtxt<'_>, pat_id: HirId) -> LetSource {
-    let hir = tcx.hir();
-
-    let parent = hir.parent_id(pat_id);
-    let_source_parent(tcx, parent, Some(pat_id))
-}
-
-fn let_source_parent(tcx: TyCtxt<'_>, parent: HirId, pat_id: Option<HirId>) -> LetSource {
-    let hir = tcx.hir();
-
-    let parent_node = hir.get(parent);
-
-    match parent_node {
-        hir::Node::Arm(hir::Arm {
-            guard: Some(hir::Guard::IfLet(&hir::Let { pat: hir::Pat { hir_id, .. }, .. })),
-            ..
-        }) if Some(*hir_id) == pat_id => {
-            return LetSource::IfLetGuard;
-        }
-        _ => {}
-    }
-
-    let parent_parent = hir.parent_id(parent);
-    let parent_parent_node = hir.get(parent_parent);
-    match parent_parent_node {
-        hir::Node::Stmt(hir::Stmt { kind: hir::StmtKind::Local(_), .. }) => {
-            return LetSource::LetElse;
-        }
-        hir::Node::Arm(hir::Arm { guard: Some(hir::Guard::If(_)), .. }) => {
-            return LetSource::IfLetGuard;
-        }
-        _ => {}
+        sess.emit_err(MovedWhileBorrowed { span: pat.span, occurrences });
     }
-
-    let parent_parent_parent = hir.parent_id(parent_parent);
-    let parent_parent_parent_parent = hir.parent_id(parent_parent_parent);
-    let parent_parent_parent_parent_node = hir.get(parent_parent_parent_parent);
-
-    if let hir::Node::Expr(hir::Expr {
-        kind: hir::ExprKind::Loop(_, _, hir::LoopSource::While, _),
-        ..
-    }) = parent_parent_parent_parent_node
-    {
-        return LetSource::WhileLet;
-    }
-
-    if let hir::Node::Expr(hir::Expr { kind: hir::ExprKind::If(..), .. }) = parent_parent_node {
-        return LetSource::IfLet;
-    }
-
-    LetSource::GenericLet
 }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index ff88d0013..32d0404bd 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -1,14 +1,15 @@
 use rustc_hir as hir;
 use rustc_index::vec::Idx;
 use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
-use rustc_middle::mir::{self, Field};
+use rustc_infer::traits::Obligation;
+use rustc_middle::mir;
 use rustc_middle::thir::{FieldPat, Pat, PatKind};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_session::lint;
 use rustc_span::Span;
-use rustc_trait_selection::traits::predicate_for_trait_def;
+use rustc_target::abi::FieldIdx;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
-use rustc_trait_selection::traits::{self, ObligationCause, PredicateObligation};
+use rustc_trait_selection::traits::{self, ObligationCause};
 
 use std::cell::Cell;
 
@@ -58,8 +59,6 @@ struct ConstToPat<'tcx> {
     // inference context used for checking `T: Structural` bounds.
     infcx: InferCtxt<'tcx>,
 
-    include_lint_checks: bool,
-
     treat_byte_string_as_slice: bool,
 }
 
@@ -92,7 +91,6 @@ impl<'tcx> ConstToPat<'tcx> {
             span,
             infcx,
             param_env: pat_ctxt.param_env,
-            include_lint_checks: pat_ctxt.include_lint_checks,
             saw_const_match_error: Cell::new(false),
             saw_const_match_lint: Cell::new(false),
             behind_reference: Cell::new(false),
@@ -133,7 +131,7 @@ impl<'tcx> ConstToPat<'tcx> {
                 })
             });
 
-        if self.include_lint_checks && !self.saw_const_match_error.get() {
+        if !self.saw_const_match_error.get() {
             // If we were able to successfully convert the const to some pat,
             // double-check that all types in the const implement `Structural`.
 
@@ -189,17 +187,15 @@ impl<'tcx> ConstToPat<'tcx> {
         // using `PartialEq::eq` in this scenario in the past.)
         let partial_eq_trait_id =
             self.tcx().require_lang_item(hir::LangItem::PartialEq, Some(self.span));
-        let obligation: PredicateObligation<'_> = predicate_for_trait_def(
+        let partial_eq_obligation = Obligation::new(
             self.tcx(),
+            ObligationCause::dummy(),
             self.param_env,
-            ObligationCause::misc(self.span, self.id.owner.def_id),
-            partial_eq_trait_id,
-            0,
-            [ty, ty],
+            self.tcx().mk_trait_ref(partial_eq_trait_id, [ty, ty]),
         );
-        // FIXME: should this call a `predicate_must_hold` variant instead?
 
-        let has_impl = self.infcx.predicate_may_hold(&obligation);
+        // FIXME: should this call a `predicate_must_hold` variant instead?
+        let has_impl = self.infcx.predicate_may_hold(&partial_eq_obligation);
 
         // Note: To fix rust-lang/rust#65466, we could just remove this type
         // walk hack for function pointers, and unconditionally error
@@ -220,7 +216,7 @@ impl<'tcx> ConstToPat<'tcx> {
     ) -> Result<Vec<FieldPat<'tcx>>, FallbackToConstRef> {
         vals.enumerate()
             .map(|(idx, val)| {
-                let field = Field::new(idx);
+                let field = FieldIdx::new(idx);
                 Ok(FieldPat { field, pattern: self.recur(val, false)? })
             })
             .collect()
@@ -240,21 +236,19 @@ impl<'tcx> ConstToPat<'tcx> {
 
         let kind = match cv.ty().kind() {
             ty::Float(_) => {
-                if self.include_lint_checks {
                     tcx.emit_spanned_lint(
                         lint::builtin::ILLEGAL_FLOATING_POINT_LITERAL_PATTERN,
                         id,
                         span,
                         FloatPattern,
                     );
-                }
                 PatKind::Constant { value: cv }
             }
             ty::Adt(adt_def, _) if adt_def.is_union() => {
                 // Matching on union fields is unsafe, we can't hide it in constants
                 self.saw_const_match_error.set(true);
                 let err = UnionPattern { span };
-                tcx.sess.create_err(err).emit_unless(!self.include_lint_checks);
+                tcx.sess.emit_err(err);
                 PatKind::Wild
             }
             ty::Adt(..)
@@ -268,7 +262,7 @@ impl<'tcx> ConstToPat<'tcx> {
             {
                 self.saw_const_match_error.set(true);
                 let err = TypeNotStructural { span, non_sm_ty };
-                tcx.sess.create_err(err).emit_unless(!self.include_lint_checks);
+                tcx.sess.emit_err(err);
                 PatKind::Wild
             }
             // If the type is not structurally comparable, just emit the constant directly,
@@ -281,8 +275,7 @@ impl<'tcx> ConstToPat<'tcx> {
             // Backwards compatibility hack because we can't cause hard errors on these
             // types, so we compare them via `PartialEq::eq` at runtime.
             ty::Adt(..) if !self.type_marked_structural(cv.ty()) && self.behind_reference.get() => {
-                if self.include_lint_checks
-                    && !self.saw_const_match_error.get()
+                if !self.saw_const_match_error.get()
                     && !self.saw_const_match_lint.get()
                 {
                     self.saw_const_match_lint.set(true);
@@ -306,7 +299,7 @@ impl<'tcx> ConstToPat<'tcx> {
                 );
                 self.saw_const_match_error.set(true);
                 let err = TypeNotStructural { span, non_sm_ty: cv.ty() };
-                tcx.sess.create_err(err).emit_unless(!self.include_lint_checks);
+                tcx.sess.emit_err(err);
                 PatKind::Wild
             }
             ty::Adt(adt_def, substs) if adt_def.is_enum() => {
@@ -340,7 +333,7 @@ impl<'tcx> ConstToPat<'tcx> {
                 ty::Dynamic(..) => {
                     self.saw_const_match_error.set(true);
                     let err = InvalidPattern { span, non_sm_ty: cv.ty() };
-                    tcx.sess.create_err(err).emit_unless(!self.include_lint_checks);
+                    tcx.sess.emit_err(err);
                     PatKind::Wild
                 }
                 // `&str` is represented as `ConstValue::Slice`, let's keep using this
@@ -407,8 +400,7 @@ impl<'tcx> ConstToPat<'tcx> {
                 // to figure out how to get a reference again.
                 ty::Adt(_, _) if !self.type_marked_structural(*pointee_ty) => {
                     if self.behind_reference.get() {
-                        if self.include_lint_checks
-                            && !self.saw_const_match_error.get()
+                        if !self.saw_const_match_error.get()
                             && !self.saw_const_match_lint.get()
                         {
                            self.saw_const_match_lint.set(true);
@@ -424,7 +416,7 @@ impl<'tcx> ConstToPat<'tcx> {
                         if !self.saw_const_match_error.get() {
                             self.saw_const_match_error.set(true);
                             let err = TypeNotStructural { span, non_sm_ty: *pointee_ty };
-                            tcx.sess.create_err(err).emit_unless(!self.include_lint_checks);
+                            tcx.sess.emit_err(err);
                         }
                         PatKind::Wild
                     }
@@ -438,7 +430,7 @@ impl<'tcx> ConstToPat<'tcx> {
                         // (except slices, which are handled in a separate arm above).
 
                         let err = UnsizedPattern { span, non_sm_ty: *pointee_ty };
-                        tcx.sess.create_err(err).emit_unless(!self.include_lint_checks);
+                        tcx.sess.emit_err(err);
 
                         PatKind::Wild
                     } else {
@@ -466,8 +458,7 @@ impl<'tcx> ConstToPat<'tcx> {
             // compilation choices change the runtime behaviour of the match.
             // See https://github.com/rust-lang/rust/issues/70861 for examples.
             ty::FnPtr(..) | ty::RawPtr(..) => {
-                if self.include_lint_checks
-                    && !self.saw_const_match_error.get()
+                if !self.saw_const_match_error.get()
                     && !self.saw_const_match_lint.get()
                 {
                     self.saw_const_match_lint.set(true);
@@ -483,13 +474,12 @@ impl<'tcx> ConstToPat<'tcx> {
             _ => {
                 self.saw_const_match_error.set(true);
                 let err = InvalidPattern { span, non_sm_ty: cv.ty() };
-                    tcx.sess.create_err(err).emit_unless(!self.include_lint_checks);
+                    tcx.sess.emit_err(err);
                 PatKind::Wild
             }
         };
 
-        if self.include_lint_checks
-            && !self.saw_const_match_error.get()
+        if !self.saw_const_match_error.get()
             && !self.saw_const_match_lint.get()
             && mir_structural_match_violation
             // FIXME(#73448): Find a way to bring const qualification into parity with
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index e5b7d685c..7c2919644 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -53,14 +53,14 @@ use smallvec::{smallvec, SmallVec};
 use rustc_data_structures::captures::Captures;
 use rustc_hir::{HirId, RangeEnd};
 use rustc_index::vec::Idx;
-use rustc_middle::mir::{self, Field};
+use rustc_middle::mir;
 use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange};
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::{self, Ty, TyCtxt, VariantDef};
 use rustc_middle::{middle::stability::EvalResult, mir::interpret::ConstValue};
 use rustc_session::lint;
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::{Integer, Size, VariantIdx};
+use rustc_target::abi::{FieldIdx, Integer, Size, VariantIdx, FIRST_VARIANT};
 
 use self::Constructor::*;
 use self::SliceKind::*;
@@ -258,7 +258,7 @@ impl IntRange {
         pcx: &PatCtxt<'_, 'p, 'tcx>,
         pats: impl Iterator<Item = &'a DeconstructedPat<'p, 'tcx>>,
         column_count: usize,
-        hir_id: HirId,
+        lint_root: HirId,
     ) {
         if self.is_singleton() {
             return;
@@ -290,7 +290,7 @@ impl IntRange {
         if !overlap.is_empty() {
             pcx.cx.tcx.emit_spanned_lint(
                 lint::builtin::OVERLAPPING_RANGE_ENDPOINTS,
-                hir_id,
+                lint_root,
                 pcx.span,
                 OverlappingRangeEndpoints { overlap, range: pcx.span },
             );
@@ -706,7 +706,7 @@ impl<'tcx> Constructor<'tcx> {
             Variant(idx) => idx,
             Single => {
                 assert!(!adt.is_enum());
-                VariantIdx::new(0)
+                FIRST_VARIANT
             }
             _ => bug!("bad constructor {:?} for adt {:?}", self, adt),
         }
@@ -1126,7 +1126,7 @@ impl<'tcx> SplitWildcard<'tcx> {
 /// Note that the number of fields of a constructor may not match the fields declared in the
 /// original struct/variant. This happens if a private or `non_exhaustive` field is uninhabited,
 /// because the code mustn't observe that it is uninhabited. In that case that field is not
-/// included in `fields`. For that reason, when you have a `mir::Field` you must use
+/// included in `fields`. For that reason, when you have a `FieldIdx` you must use
 /// `index_with_declared_idx`.
 #[derive(Debug, Clone, Copy)]
 pub(super) struct Fields<'p, 'tcx> {
@@ -1154,8 +1154,9 @@ impl<'p, 'tcx> Fields<'p, 'tcx> {
     fn wildcards_from_tys(
         cx: &MatchCheckCtxt<'p, 'tcx>,
         tys: impl IntoIterator<Item = Ty<'tcx>>,
+        span: Span,
     ) -> Self {
-        Fields::from_iter(cx, tys.into_iter().map(DeconstructedPat::wildcard))
+        Fields::from_iter(cx, tys.into_iter().map(|ty| DeconstructedPat::wildcard(ty, span)))
     }
 
     // In the cases of either a `#[non_exhaustive]` field list or a non-public field, we hide
@@ -1165,7 +1166,7 @@ impl<'p, 'tcx> Fields<'p, 'tcx> {
         cx: &'a MatchCheckCtxt<'p, 'tcx>,
         ty: Ty<'tcx>,
         variant: &'a VariantDef,
-    ) -> impl Iterator<Item = (Field, Ty<'tcx>)> + Captures<'a> + Captures<'p> {
+    ) -> impl Iterator<Item = (FieldIdx, Ty<'tcx>)> + Captures<'a> + Captures<'p> {
         let ty::Adt(adt, substs) = ty.kind() else { bug!() };
         // Whether we must not match the fields of this variant exhaustively.
         let is_non_exhaustive = variant.is_field_list_non_exhaustive() && !adt.did().is_local();
@@ -1180,7 +1181,7 @@ impl<'p, 'tcx> Fields<'p, 'tcx> {
             if is_uninhabited && (!is_visible || is_non_exhaustive) {
                 None
             } else {
-                Some((Field::new(i), ty))
+                Some((FieldIdx::new(i), ty))
             }
         })
     }
@@ -1191,18 +1192,18 @@ impl<'p, 'tcx> Fields<'p, 'tcx> {
     pub(super) fn wildcards(pcx: &PatCtxt<'_, 'p, 'tcx>, constructor: &Constructor<'tcx>) -> Self {
         let ret = match constructor {
             Single | Variant(_) => match pcx.ty.kind() {
-                ty::Tuple(fs) => Fields::wildcards_from_tys(pcx.cx, fs.iter()),
-                ty::Ref(_, rty, _) => Fields::wildcards_from_tys(pcx.cx, once(*rty)),
+                ty::Tuple(fs) => Fields::wildcards_from_tys(pcx.cx, fs.iter(), pcx.span),
+                ty::Ref(_, rty, _) => Fields::wildcards_from_tys(pcx.cx, once(*rty), pcx.span),
                 ty::Adt(adt, substs) => {
                     if adt.is_box() {
                         // The only legal patterns of type `Box` (outside `std`) are `_` and box
                         // patterns. If we're here we can assume this is a box pattern.
-                        Fields::wildcards_from_tys(pcx.cx, once(substs.type_at(0)))
+                        Fields::wildcards_from_tys(pcx.cx, once(substs.type_at(0)), pcx.span)
                     } else {
                         let variant = &adt.variant(constructor.variant_index_for_adt(*adt));
                         let tys = Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant)
                             .map(|(_, ty)| ty);
-                        Fields::wildcards_from_tys(pcx.cx, tys)
+                        Fields::wildcards_from_tys(pcx.cx, tys, pcx.span)
                     }
                 }
                 _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx),
@@ -1210,7 +1211,7 @@ impl<'p, 'tcx> Fields<'p, 'tcx> {
             Slice(slice) => match *pcx.ty.kind() {
                 ty::Slice(ty) | ty::Array(ty, _) => {
                     let arity = slice.arity();
-                    Fields::wildcards_from_tys(pcx.cx, (0..arity).map(|_| ty))
+                    Fields::wildcards_from_tys(pcx.cx, (0..arity).map(|_| ty), pcx.span)
                 }
                 _ => bug!("bad slice pattern {:?} {:?}", constructor, pcx),
             },
@@ -1251,8 +1252,8 @@ pub(crate) struct DeconstructedPat<'p, 'tcx> {
 }
 
 impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
-    pub(super) fn wildcard(ty: Ty<'tcx>) -> Self {
-        Self::new(Wildcard, Fields::empty(), ty, DUMMY_SP)
+    pub(super) fn wildcard(ty: Ty<'tcx>, span: Span) -> Self {
+        Self::new(Wildcard, Fields::empty(), ty, span)
     }
 
     pub(super) fn new(
@@ -1269,7 +1270,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
     /// `Some(_)`.
     pub(super) fn wild_from_ctor(pcx: &PatCtxt<'_, 'p, 'tcx>, ctor: Constructor<'tcx>) -> Self {
         let fields = Fields::wildcards(pcx, &ctor);
-        DeconstructedPat::new(ctor, fields, pcx.ty, DUMMY_SP)
+        DeconstructedPat::new(ctor, fields, pcx.ty, pcx.span)
     }
 
     /// Clone this value. This method emphasizes that cloning loses reachability information and
@@ -1298,7 +1299,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                     ty::Tuple(fs) => {
                         ctor = Single;
                         let mut wilds: SmallVec<[_; 2]> =
-                            fs.iter().map(DeconstructedPat::wildcard).collect();
+                            fs.iter().map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
                         for pat in subpatterns {
                             wilds[pat.field.index()] = mkpat(&pat.pattern);
                         }
@@ -1317,11 +1318,11 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                         // normally or through box-patterns. We'll have to figure out a proper
                         // solution when we introduce generalized deref patterns. Also need to
                         // prevent mixing of those two options.
-                        let pat = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
-                        let pat = if let Some(pat) = pat {
+                        let pattern = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
+                        let pat = if let Some(pat) = pattern {
                             mkpat(&pat.pattern)
                         } else {
-                            DeconstructedPat::wildcard(substs.type_at(0))
+                            DeconstructedPat::wildcard(substs.type_at(0), pat.span)
                         };
                         ctor = Single;
                         fields = Fields::singleton(cx, pat);
@@ -1343,7 +1344,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                                 ty
                             });
                         let mut wilds: SmallVec<[_; 2]> =
-                            tys.map(DeconstructedPat::wildcard).collect();
+                            tys.map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
                         for pat in subpatterns {
                             if let Some(i) = field_id_to_id[pat.field.index()] {
                                 wilds[i] = mkpat(&pat.pattern);
@@ -1438,7 +1439,7 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                 ty::Tuple(..) => PatKind::Leaf {
                     subpatterns: subpatterns
                         .enumerate()
-                        .map(|(i, pattern)| FieldPat { field: Field::new(i), pattern })
+                        .map(|(i, pattern)| FieldPat { field: FieldIdx::new(i), pattern })
                         .collect(),
                 },
                 ty::Adt(adt_def, _) if adt_def.is_box() => {
@@ -1566,8 +1567,10 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                         };
                         let prefix = &self.fields.fields[..prefix];
                         let suffix = &self.fields.fields[self_slice.arity() - suffix..];
-                        let wildcard: &_ =
-                            pcx.cx.pattern_arena.alloc(DeconstructedPat::wildcard(inner_ty));
+                        let wildcard: &_ = pcx
+                            .cx
+                            .pattern_arena
+                            .alloc(DeconstructedPat::wildcard(inner_ty, pcx.span));
                         let extra_wildcards = other_slice.arity() - self_slice.arity();
                         let extra_wildcards = (0..extra_wildcards).map(|_| wildcard);
                         prefix.iter().chain(extra_wildcards).chain(suffix).collect()
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index 41306dd80..70d015a39 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -21,29 +21,20 @@ use rustc_middle::mir::interpret::{
     ConstValue, ErrorHandled, LitToConstError, LitToConstInput, Scalar,
 };
 use rustc_middle::mir::{self, UserTypeProjection};
-use rustc_middle::mir::{BorrowKind, Field, Mutability};
+use rustc_middle::mir::{BorrowKind, Mutability};
 use rustc_middle::thir::{Ascription, BindingMode, FieldPat, LocalVarId, Pat, PatKind, PatRange};
 use rustc_middle::ty::subst::{GenericArg, SubstsRef};
 use rustc_middle::ty::CanonicalUserTypeAnnotation;
-use rustc_middle::ty::{self, AdtDef, ConstKind, DefIdTree, Region, Ty, TyCtxt, UserType};
+use rustc_middle::ty::{self, AdtDef, ConstKind, Region, Ty, TyCtxt, UserType};
 use rustc_span::{Span, Symbol};
+use rustc_target::abi::FieldIdx;
 
 use std::cmp::Ordering;
 
-#[derive(Clone, Debug)]
-enum PatternError {
-    AssocConstInPattern(Span),
-    ConstParamInPattern(Span),
-    StaticInPattern(Span),
-    NonConstPath(Span),
-}
-
 struct PatCtxt<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     typeck_results: &'a ty::TypeckResults<'tcx>,
-    errors: Vec<PatternError>,
-    include_lint_checks: bool,
 }
 
 pub(super) fn pat_from_hir<'a, 'tcx>(
@@ -52,30 +43,13 @@ pub(super) fn pat_from_hir<'a, 'tcx>(
     typeck_results: &'a ty::TypeckResults<'tcx>,
     pat: &'tcx hir::Pat<'tcx>,
 ) -> Box<Pat<'tcx>> {
-    let mut pcx = PatCtxt::new(tcx, param_env, typeck_results);
+    let mut pcx = PatCtxt { tcx, param_env, typeck_results };
     let result = pcx.lower_pattern(pat);
-    if !pcx.errors.is_empty() {
-        let msg = format!("encountered errors lowering pattern: {:?}", pcx.errors);
-        tcx.sess.delay_span_bug(pat.span, &msg);
-    }
     debug!("pat_from_hir({:?}) = {:?}", pat, result);
     result
 }
 
 impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
-    fn new(
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-        typeck_results: &'a ty::TypeckResults<'tcx>,
-    ) -> Self {
-        PatCtxt { tcx, param_env, typeck_results, errors: vec![], include_lint_checks: false }
-    }
-
-    fn include_lint_checks(&mut self) -> &mut Self {
-        self.include_lint_checks = true;
-        self
-    }
-
     fn lower_pattern(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Box<Pat<'tcx>> {
         // When implicit dereferences have been inserted in this pattern, the unadjusted lowered
         // pattern has the type that results *after* dereferencing. For example, in this code:
@@ -356,7 +330,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 let subpatterns = fields
                     .iter()
                     .map(|field| FieldPat {
-                        field: Field::new(self.typeck_results.field_index(field.hir_id)),
+                        field: self.typeck_results.field_index(field.hir_id),
                         pattern: self.lower_pattern(&field.pat),
                     })
                     .collect();
@@ -379,7 +353,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         pats.iter()
             .enumerate_and_adjust(expected_len, gap_pos)
             .map(|(i, subpattern)| FieldPat {
-                field: Field::new(i),
+                field: FieldIdx::new(i),
                 pattern: self.lower_pattern(subpattern),
             })
             .collect()
@@ -472,12 +446,15 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             | Res::SelfTyAlias { .. }
             | Res::SelfCtor(..) => PatKind::Leaf { subpatterns },
             _ => {
-                let pattern_error = match res {
-                    Res::Def(DefKind::ConstParam, _) => PatternError::ConstParamInPattern(span),
-                    Res::Def(DefKind::Static(_), _) => PatternError::StaticInPattern(span),
-                    _ => PatternError::NonConstPath(span),
+                match res {
+                    Res::Def(DefKind::ConstParam, _) => {
+                        self.tcx.sess.emit_err(ConstParamInPattern { span })
+                    }
+                    Res::Def(DefKind::Static(_), _) => {
+                        self.tcx.sess.emit_err(StaticInPattern { span })
+                    }
+                    _ => self.tcx.sess.emit_err(NonConstPath { span }),
                 };
-                self.errors.push(pattern_error);
                 PatKind::Wild
             }
         };
@@ -530,7 +507,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 // It should be assoc consts if there's no error but we cannot resolve it.
                 debug_assert!(is_associated_const);
 
-                self.errors.push(PatternError::AssocConstInPattern(span));
+                self.tcx.sess.emit_err(AssocConstInPattern { span });
 
                 return pat_from_kind(PatKind::Wild);
             }
@@ -608,7 +585,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         match value {
             mir::ConstantKind::Ty(c) => match c.kind() {
                 ConstKind::Param(_) => {
-                    self.errors.push(PatternError::ConstParamInPattern(span));
+                    self.tcx.sess.emit_err(ConstParamInPattern { span });
                     return PatKind::Wild;
                 }
                 ConstKind::Error(_) => {
@@ -723,7 +700,7 @@ macro_rules! ClonePatternFoldableImpls {
 }
 
 ClonePatternFoldableImpls! { <'tcx>
-    Span, Field, Mutability, Symbol, LocalVarId, usize,
+    Span, FieldIdx, Mutability, Symbol, LocalVarId, usize,
     Region<'tcx>, Ty<'tcx>, BindingMode, AdtDef<'tcx>,
     SubstsRef<'tcx>, &'tcx GenericArg<'tcx>, UserType<'tcx>,
     UserTypeProjection, CanonicalUserTypeAnnotation<'tcx>
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index be66d0d47..d8f66a175 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -318,6 +318,8 @@ pub(crate) struct MatchCheckCtxt<'p, 'tcx> {
     pub(crate) module: DefId,
     pub(crate) param_env: ty::ParamEnv<'tcx>,
     pub(crate) pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
+    /// Only produce `NON_EXHAUSTIVE_OMITTED_PATTERNS` lint on refutable patterns.
+    pub(crate) refutable: bool,
 }
 
 impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
@@ -603,7 +605,7 @@ impl<'p, 'tcx> Usefulness<'p, 'tcx> {
                     let new_patterns = if pcx.is_non_exhaustive {
                         // Here we don't want the user to try to list all variants, we want them to add
                         // a wildcard, so we only suggest that.
-                        vec![DeconstructedPat::wildcard(pcx.ty)]
+                        vec![DeconstructedPat::wildcard(pcx.ty, pcx.span)]
                     } else {
                         let mut split_wildcard = SplitWildcard::new(pcx);
                         split_wildcard.split(pcx, matrix.heads().map(DeconstructedPat::ctor));
@@ -630,7 +632,7 @@ impl<'p, 'tcx> Usefulness<'p, 'tcx> {
                             .collect();
 
                         if hide_variant_show_wild {
-                            new.push(DeconstructedPat::wildcard(pcx.ty));
+                            new.push(DeconstructedPat::wildcard(pcx.ty, pcx.span));
                         }
 
                         new
@@ -733,7 +735,7 @@ impl<'p, 'tcx> Witness<'p, 'tcx> {
             let arity = ctor.arity(pcx);
             let pats = self.0.drain((len - arity)..).rev();
             let fields = Fields::from_iter(pcx.cx, pats);
-            DeconstructedPat::new(ctor.clone(), fields, pcx.ty, DUMMY_SP)
+            DeconstructedPat::new(ctor.clone(), fields, pcx.ty, pcx.span)
         };
 
         self.0.push(pat);
@@ -764,13 +766,13 @@ impl<'p, 'tcx> Witness<'p, 'tcx> {
 /// `is_under_guard` is used to inform if the pattern has a guard. If it
 /// has one it must not be inserted into the matrix. This shouldn't be
 /// relied on for soundness.
-#[instrument(level = "debug", skip(cx, matrix, hir_id), ret)]
+#[instrument(level = "debug", skip(cx, matrix, lint_root), ret)]
 fn is_useful<'p, 'tcx>(
     cx: &MatchCheckCtxt<'p, 'tcx>,
     matrix: &Matrix<'p, 'tcx>,
     v: &PatStack<'p, 'tcx>,
     witness_preference: ArmType,
-    hir_id: HirId,
+    lint_root: HirId,
     is_under_guard: bool,
     is_top_level: bool,
 ) -> Usefulness<'p, 'tcx> {
@@ -803,7 +805,7 @@ fn is_useful<'p, 'tcx>(
         for v in v.expand_or_pat() {
             debug!(?v);
             let usefulness = ensure_sufficient_stack(|| {
-                is_useful(cx, &matrix, &v, witness_preference, hir_id, is_under_guard, false)
+                is_useful(cx, &matrix, &v, witness_preference, lint_root, is_under_guard, false)
             });
             debug!(?usefulness);
             ret.extend(usefulness);
@@ -836,7 +838,7 @@ fn is_useful<'p, 'tcx>(
                 pcx,
                 matrix.heads(),
                 matrix.column_count().unwrap_or(0),
-                hir_id,
+                lint_root,
             )
         }
         // We split the head constructor of `v`.
@@ -851,7 +853,15 @@ fn is_useful<'p, 'tcx>(
             let spec_matrix = start_matrix.specialize_constructor(pcx, &ctor);
             let v = v.pop_head_constructor(pcx, &ctor);
             let usefulness = ensure_sufficient_stack(|| {
-                is_useful(cx, &spec_matrix, &v, witness_preference, hir_id, is_under_guard, false)
+                is_useful(
+                    cx,
+                    &spec_matrix,
+                    &v,
+                    witness_preference,
+                    lint_root,
+                    is_under_guard,
+                    false,
+                )
             });
             let usefulness = usefulness.apply_constructor(pcx, start_matrix, &ctor);
 
@@ -859,6 +869,8 @@ fn is_useful<'p, 'tcx>(
             // that has the potential to trigger the `non_exhaustive_omitted_patterns` lint.
             // To understand the workings checkout `Constructor::split` and `SplitWildcard::new/into_ctors`
             if is_non_exhaustive_and_wild
+                // Only emit a lint on refutable patterns.
+                && cx.refutable
                 // We check that the match has a wildcard pattern and that wildcard is useful,
                 // meaning there are variants that are covered by the wildcard. Without the check
                 // for `witness_preference` the lint would trigger on `if let NonExhaustiveEnum::A = foo {}`
@@ -893,7 +905,7 @@ fn is_useful<'p, 'tcx>(
                 // NB: The partner lint for structs lives in `compiler/rustc_hir_analysis/src/check/pat.rs`.
                 cx.tcx.emit_spanned_lint(
                     NON_EXHAUSTIVE_OMITTED_PATTERNS,
-                    hir_id,
+                    lint_root,
                     pcx.span,
                     NonExhaustiveOmittedPattern {
                         scrut_ty: pcx.ty,
@@ -951,7 +963,7 @@ pub(crate) struct UsefulnessReport<'p, 'tcx> {
 pub(crate) fn compute_match_usefulness<'p, 'tcx>(
     cx: &MatchCheckCtxt<'p, 'tcx>,
     arms: &[MatchArm<'p, 'tcx>],
-    scrut_hir_id: HirId,
+    lint_root: HirId,
     scrut_ty: Ty<'tcx>,
 ) -> UsefulnessReport<'p, 'tcx> {
     let mut matrix = Matrix::empty();
@@ -974,9 +986,9 @@ pub(crate) fn compute_match_usefulness<'p, 'tcx>(
         })
         .collect();
 
-    let wild_pattern = cx.pattern_arena.alloc(DeconstructedPat::wildcard(scrut_ty));
+    let wild_pattern = cx.pattern_arena.alloc(DeconstructedPat::wildcard(scrut_ty, DUMMY_SP));
     let v = PatStack::from_pattern(wild_pattern);
-    let usefulness = is_useful(cx, &matrix, &v, FakeExtraWildcard, scrut_hir_id, false, true);
+    let usefulness = is_useful(cx, &matrix, &v, FakeExtraWildcard, lint_root, false, true);
     let non_exhaustiveness_witnesses = match usefulness {
         WithWitnesses(pats) => pats.into_iter().map(|w| w.single_pattern()).collect(),
         NoWitnesses { .. } => bug!(),
diff --git a/compiler/rustc_mir_build/src/thir/print.rs b/compiler/rustc_mir_build/src/thir/print.rs
index 8028227aa..ed61d6ee7 100644
--- a/compiler/rustc_mir_build/src/thir/print.rs
+++ b/compiler/rustc_mir_build/src/thir/print.rs
@@ -151,6 +151,7 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
                 initializer,
                 else_block,
                 lint_level,
+                span,
             } => {
                 print_indented!(self, "kind: Let {", depth_lvl + 1);
                 print_indented!(
@@ -181,6 +182,7 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
                 }
 
                 print_indented!(self, format!("lint_level: {:?}", lint_level), depth_lvl + 2);
+                print_indented!(self, format!("span: {:?}", span), depth_lvl + 2);
                 print_indented!(self, "}", depth_lvl + 1);
             }
         }
diff --git a/compiler/rustc_mir_dataflow/locales/en-US.ftl b/compiler/rustc_mir_dataflow/messages.ftl
index 988541525..988541525 100644
--- a/compiler/rustc_mir_dataflow/locales/en-US.ftl
+++ b/compiler/rustc_mir_dataflow/messages.ftl
diff --git a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs
index bd1208762..bd8ec82df 100644
--- a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs
@@ -7,7 +7,7 @@ use rustc_middle::traits::Reveal;
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::util::IntTypeExt;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT};
 use std::{fmt, iter};
 
 /// The value of an inserted drop flag.
@@ -77,10 +77,10 @@ impl Unwind {
         }
     }
 
-    fn into_option(self) -> Option<BasicBlock> {
+    fn into_action(self) -> UnwindAction {
         match self {
-            Unwind::To(bb) => Some(bb),
-            Unwind::InCleanup => None,
+            Unwind::To(bb) => UnwindAction::Cleanup(bb),
+            Unwind::InCleanup => UnwindAction::Terminate,
         }
     }
 
@@ -129,7 +129,7 @@ pub trait DropElaborator<'a, 'tcx>: fmt::Debug {
     /// Returns the subpath of a field of `path` (or `None` if there is no dedicated subpath).
     ///
     /// If this returns `None`, `field` will not get a dedicated drop flag.
-    fn field_subpath(&self, path: Self::Path, field: Field) -> Option<Self::Path>;
+    fn field_subpath(&self, path: Self::Path, field: FieldIdx) -> Option<Self::Path>;
 
     /// Returns the subpath of a dereference of `path` (or `None` if there is no dedicated subpath).
     ///
@@ -236,7 +236,7 @@ where
                     TerminatorKind::Drop {
                         place: self.place,
                         target: self.succ,
-                        unwind: self.unwind.into_option(),
+                        unwind: self.unwind.into_action(),
                     },
                 );
             }
@@ -269,7 +269,7 @@ where
             .iter()
             .enumerate()
             .map(|(i, f)| {
-                let field = Field::new(i);
+                let field = FieldIdx::new(i);
                 let subpath = self.elaborator.field_subpath(variant_path, field);
                 let tcx = self.tcx();
 
@@ -397,8 +397,8 @@ where
             .enumerate()
             .map(|(i, &ty)| {
                 (
-                    self.tcx().mk_place_field(self.place, Field::new(i), ty),
-                    self.elaborator.field_subpath(self.path, Field::new(i)),
+                    self.tcx().mk_place_field(self.place, FieldIdx::new(i), ty),
+                    self.elaborator.field_subpath(self.path, FieldIdx::new(i)),
                 )
             })
             .collect();
@@ -411,14 +411,14 @@ where
     fn open_drop_for_box(&mut self, adt: ty::AdtDef<'tcx>, substs: SubstsRef<'tcx>) -> BasicBlock {
         // drop glue is sent straight to codegen
         // box cannot be directly dereferenced
-        let unique_ty = adt.non_enum_variant().fields[0].ty(self.tcx(), substs);
-        let nonnull_ty =
-            unique_ty.ty_adt_def().unwrap().non_enum_variant().fields[0].ty(self.tcx(), substs);
+        let unique_ty = adt.non_enum_variant().fields[FieldIdx::new(0)].ty(self.tcx(), substs);
+        let unique_variant = unique_ty.ty_adt_def().unwrap().non_enum_variant();
+        let nonnull_ty = unique_variant.fields[FieldIdx::from_u32(0)].ty(self.tcx(), substs);
         let ptr_ty = self.tcx().mk_imm_ptr(substs[0].expect_ty());
 
-        let unique_place = self.tcx().mk_place_field(self.place, Field::new(0), unique_ty);
-        let nonnull_place = self.tcx().mk_place_field(unique_place, Field::new(0), nonnull_ty);
-        let ptr_place = self.tcx().mk_place_field(nonnull_place, Field::new(0), ptr_ty);
+        let unique_place = self.tcx().mk_place_field(self.place, FieldIdx::new(0), unique_ty);
+        let nonnull_place = self.tcx().mk_place_field(unique_place, FieldIdx::new(0), nonnull_ty);
+        let ptr_place = self.tcx().mk_place_field(nonnull_place, FieldIdx::new(0), ptr_ty);
         let interior = self.tcx().mk_place_deref(ptr_place);
 
         let interior_path = self.elaborator.deref_subpath(self.path);
@@ -468,7 +468,7 @@ where
             let fields = self.move_paths_for_fields(
                 self.place,
                 self.path,
-                &adt.variant(VariantIdx::new(0)),
+                &adt.variant(FIRST_VARIANT),
                 substs,
             );
             self.drop_ladder(fields, succ, unwind)
@@ -640,7 +640,7 @@ where
                     args: vec![Operand::Move(Place::from(ref_place))],
                     destination: unit_temp,
                     target: Some(succ),
-                    cleanup: unwind.into_option(),
+                    unwind: unwind.into_action(),
                     from_hir_call: true,
                     fn_span: self.source_info.span,
                 },
@@ -655,26 +655,20 @@ where
     ///
     /// ```text
     /// loop-block:
-    ///    can_go = cur == length_or_end
+    ///    can_go = cur == len
     ///    if can_go then succ else drop-block
     /// drop-block:
-    ///    if ptr_based {
-    ///        ptr = cur
-    ///        cur = cur.offset(1)
-    ///    } else {
-    ///        ptr = &raw mut P[cur]
-    ///        cur = cur + 1
-    ///    }
+    ///    ptr = &raw mut P[cur]
+    ///    cur = cur + 1
     ///    drop(ptr)
     /// ```
     fn drop_loop(
         &mut self,
         succ: BasicBlock,
         cur: Local,
-        length_or_end: Place<'tcx>,
+        len: Local,
         ety: Ty<'tcx>,
         unwind: Unwind,
-        ptr_based: bool,
     ) -> BasicBlock {
         let copy = |place: Place<'tcx>| Operand::Copy(place);
         let move_ = |place: Place<'tcx>| Operand::Move(place);
@@ -683,22 +677,19 @@ where
         let ptr_ty = tcx.mk_ptr(ty::TypeAndMut { ty: ety, mutbl: hir::Mutability::Mut });
         let ptr = Place::from(self.new_temp(ptr_ty));
         let can_go = Place::from(self.new_temp(tcx.types.bool));
-
         let one = self.constant_usize(1);
-        let (ptr_next, cur_next) = if ptr_based {
-            (
-                Rvalue::Use(copy(cur.into())),
-                Rvalue::BinaryOp(BinOp::Offset, Box::new((move_(cur.into()), one))),
-            )
-        } else {
-            (
-                Rvalue::AddressOf(Mutability::Mut, tcx.mk_place_index(self.place, cur)),
-                Rvalue::BinaryOp(BinOp::Add, Box::new((move_(cur.into()), one))),
-            )
-        };
 
         let drop_block = BasicBlockData {
-            statements: vec![self.assign(ptr, ptr_next), self.assign(Place::from(cur), cur_next)],
+            statements: vec![
+                self.assign(
+                    ptr,
+                    Rvalue::AddressOf(Mutability::Mut, tcx.mk_place_index(self.place, cur)),
+                ),
+                self.assign(
+                    cur.into(),
+                    Rvalue::BinaryOp(BinOp::Add, Box::new((move_(cur.into()), one))),
+                ),
+            ],
             is_cleanup: unwind.is_cleanup(),
             terminator: Some(Terminator {
                 source_info: self.source_info,
@@ -711,10 +702,7 @@ where
         let loop_block = BasicBlockData {
             statements: vec![self.assign(
                 can_go,
-                Rvalue::BinaryOp(
-                    BinOp::Eq,
-                    Box::new((copy(Place::from(cur)), copy(length_or_end))),
-                ),
+                Rvalue::BinaryOp(BinOp::Eq, Box::new((copy(Place::from(cur)), copy(len.into())))),
             )],
             is_cleanup: unwind.is_cleanup(),
             terminator: Some(Terminator {
@@ -729,7 +717,7 @@ where
             TerminatorKind::Drop {
                 place: tcx.mk_place_deref(ptr),
                 target: loop_block,
-                unwind: unwind.into_option(),
+                unwind: unwind.into_action(),
             },
         );
 
@@ -738,118 +726,97 @@ where
 
     fn open_drop_for_array(&mut self, ety: Ty<'tcx>, opt_size: Option<u64>) -> BasicBlock {
         debug!("open_drop_for_array({:?}, {:?})", ety, opt_size);
-
-        // if size_of::<ety>() == 0 {
-        //     index_based_loop
-        // } else {
-        //     ptr_based_loop
-        // }
-
         let tcx = self.tcx();
 
         if let Some(size) = opt_size {
-            let fields: Vec<(Place<'tcx>, Option<D::Path>)> = (0..size)
-                .map(|i| {
-                    (
-                        tcx.mk_place_elem(
-                            self.place,
-                            ProjectionElem::ConstantIndex {
-                                offset: i,
-                                min_length: size,
-                                from_end: false,
-                            },
-                        ),
-                        self.elaborator.array_subpath(self.path, i, size),
-                    )
-                })
-                .collect();
-
-            if fields.iter().any(|(_, path)| path.is_some()) {
+            enum ProjectionKind<Path> {
+                Drop(std::ops::Range<u64>),
+                Keep(u64, Path),
+            }
+            // Previously, we'd make a projection for every element in the array and create a drop
+            // ladder if any `array_subpath` was `Some`, i.e. moving out with an array pattern.
+            // This caused huge memory usage when generating the drops for large arrays, so we instead
+            // record the *subslices* which are dropped and the *indexes* which are kept
+            let mut drop_ranges = vec![];
+            let mut dropping = true;
+            let mut start = 0;
+            for i in 0..size {
+                let path = self.elaborator.array_subpath(self.path, i, size);
+                if dropping && path.is_some() {
+                    drop_ranges.push(ProjectionKind::Drop(start..i));
+                    dropping = false;
+                } else if !dropping && path.is_none() {
+                    dropping = true;
+                    start = i;
+                }
+                if let Some(path) = path {
+                    drop_ranges.push(ProjectionKind::Keep(i, path));
+                }
+            }
+            if !drop_ranges.is_empty() {
+                if dropping {
+                    drop_ranges.push(ProjectionKind::Drop(start..size));
+                }
+                let fields = drop_ranges
+                    .iter()
+                    .rev()
+                    .map(|p| {
+                        let (project, path) = match p {
+                            ProjectionKind::Drop(r) => (
+                                ProjectionElem::Subslice {
+                                    from: r.start,
+                                    to: r.end,
+                                    from_end: false,
+                                },
+                                None,
+                            ),
+                            &ProjectionKind::Keep(offset, path) => (
+                                ProjectionElem::ConstantIndex {
+                                    offset,
+                                    min_length: size,
+                                    from_end: false,
+                                },
+                                Some(path),
+                            ),
+                        };
+                        (tcx.mk_place_elem(self.place, project), path)
+                    })
+                    .collect::<Vec<_>>();
                 let (succ, unwind) = self.drop_ladder_bottom();
                 return self.drop_ladder(fields, succ, unwind).0;
             }
         }
 
-        let move_ = |place: Place<'tcx>| Operand::Move(place);
-        let elem_size = Place::from(self.new_temp(tcx.types.usize));
-        let len = Place::from(self.new_temp(tcx.types.usize));
-
-        let base_block = BasicBlockData {
-            statements: vec![
-                self.assign(elem_size, Rvalue::NullaryOp(NullOp::SizeOf, ety)),
-                self.assign(len, Rvalue::Len(self.place)),
-            ],
-            is_cleanup: self.unwind.is_cleanup(),
-            terminator: Some(Terminator {
-                source_info: self.source_info,
-                kind: TerminatorKind::SwitchInt {
-                    discr: move_(elem_size),
-                    targets: SwitchTargets::static_if(
-                        0,
-                        self.drop_loop_pair(ety, false, len),
-                        self.drop_loop_pair(ety, true, len),
-                    ),
-                },
-            }),
-        };
-        self.elaborator.patch().new_block(base_block)
+        self.drop_loop_pair(ety)
     }
 
     /// Creates a pair of drop-loops of `place`, which drops its contents, even
-    /// in the case of 1 panic. If `ptr_based`, creates a pointer loop,
-    /// otherwise create an index loop.
-    fn drop_loop_pair(
-        &mut self,
-        ety: Ty<'tcx>,
-        ptr_based: bool,
-        length: Place<'tcx>,
-    ) -> BasicBlock {
-        debug!("drop_loop_pair({:?}, {:?})", ety, ptr_based);
+    /// in the case of 1 panic.
+    fn drop_loop_pair(&mut self, ety: Ty<'tcx>) -> BasicBlock {
+        debug!("drop_loop_pair({:?})", ety);
         let tcx = self.tcx();
-        let iter_ty = if ptr_based { tcx.mk_mut_ptr(ety) } else { tcx.types.usize };
+        let len = self.new_temp(tcx.types.usize);
+        let cur = self.new_temp(tcx.types.usize);
 
-        let cur = self.new_temp(iter_ty);
-        let length_or_end = if ptr_based { Place::from(self.new_temp(iter_ty)) } else { length };
+        let unwind =
+            self.unwind.map(|unwind| self.drop_loop(unwind, cur, len, ety, Unwind::InCleanup));
 
-        let unwind = self.unwind.map(|unwind| {
-            self.drop_loop(unwind, cur, length_or_end, ety, Unwind::InCleanup, ptr_based)
-        });
+        let loop_block = self.drop_loop(self.succ, cur, len, ety, unwind);
 
-        let loop_block = self.drop_loop(self.succ, cur, length_or_end, ety, unwind, ptr_based);
-
-        let cur = Place::from(cur);
-        let drop_block_stmts = if ptr_based {
-            let tmp_ty = tcx.mk_mut_ptr(self.place_ty(self.place));
-            let tmp = Place::from(self.new_temp(tmp_ty));
-            // tmp = &raw mut P;
-            // cur = tmp as *mut T;
-            // end = Offset(cur, len);
-            let mir_cast_kind = ty::cast::mir_cast_kind(iter_ty, tmp_ty);
-            vec![
-                self.assign(tmp, Rvalue::AddressOf(Mutability::Mut, self.place)),
-                self.assign(cur, Rvalue::Cast(mir_cast_kind, Operand::Move(tmp), iter_ty)),
-                self.assign(
-                    length_or_end,
-                    Rvalue::BinaryOp(
-                        BinOp::Offset,
-                        Box::new((Operand::Copy(cur), Operand::Move(length))),
-                    ),
-                ),
-            ]
-        } else {
-            // cur = 0 (length already pushed)
-            let zero = self.constant_usize(0);
-            vec![self.assign(cur, Rvalue::Use(zero))]
-        };
-        let drop_block = self.elaborator.patch().new_block(BasicBlockData {
-            statements: drop_block_stmts,
+        let zero = self.constant_usize(0);
+        let block = BasicBlockData {
+            statements: vec![
+                self.assign(len.into(), Rvalue::Len(self.place)),
+                self.assign(cur.into(), Rvalue::Use(zero)),
+            ],
             is_cleanup: unwind.is_cleanup(),
             terminator: Some(Terminator {
                 source_info: self.source_info,
                 kind: TerminatorKind::Goto { target: loop_block },
             }),
-        });
+        };
 
+        let drop_block = self.elaborator.patch().new_block(block);
         // FIXME(#34708): handle partially-dropped array/slice elements.
         let reset_block = self.drop_flag_reset_block(DropFlagMode::Deep, drop_block, unwind);
         self.drop_flag_test_block(reset_block, self.succ, unwind)
@@ -893,7 +860,7 @@ where
                 let size = size.try_eval_target_usize(self.tcx(), self.elaborator.param_env());
                 self.open_drop_for_array(*ety, size)
             }
-            ty::Slice(ety) => self.open_drop_for_array(*ety, None),
+            ty::Slice(ety) => self.drop_loop_pair(*ety),
 
             _ => span_bug!(self.source_info.span, "open drop from non-ADT `{:?}`", ty),
         }
@@ -963,12 +930,12 @@ where
         let unit_temp = Place::from(self.new_temp(tcx.mk_unit()));
         let free_func = tcx.require_lang_item(LangItem::BoxFree, Some(self.source_info.span));
         let args = adt
-            .variant(VariantIdx::new(0))
+            .variant(FIRST_VARIANT)
             .fields
             .iter()
             .enumerate()
             .map(|(i, f)| {
-                let field = Field::new(i);
+                let field = FieldIdx::new(i);
                 let field_ty = f.ty(tcx, substs);
                 Operand::Move(tcx.mk_place_field(self.place, field, field_ty))
             })
@@ -979,7 +946,11 @@ where
             args,
             destination: unit_temp,
             target: Some(target),
-            cleanup: None,
+            unwind: if unwind.is_cleanup() {
+                UnwindAction::Terminate
+            } else {
+                UnwindAction::Continue
+            },
             from_hir_call: false,
             fn_span: self.source_info.span,
         }; // FIXME(#43234)
@@ -992,7 +963,7 @@ where
 
     fn drop_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock {
         let block =
-            TerminatorKind::Drop { place: self.place, target, unwind: unwind.into_option() };
+            TerminatorKind::Drop { place: self.place, target, unwind: unwind.into_action() };
         self.new_block(unwind, block)
     }
 
diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs
index 2ae3ae02f..c8fe1af66 100644
--- a/compiler/rustc_mir_dataflow/src/framework/direction.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs
@@ -1,4 +1,4 @@
-use rustc_middle::mir::{self, BasicBlock, Location, SwitchTargets};
+use rustc_middle::mir::{self, BasicBlock, Location, SwitchTargets, UnwindAction};
 use rustc_middle::ty::TyCtxt;
 use std::ops::RangeInclusive;
 
@@ -474,15 +474,14 @@ impl Direction for Forward {
     {
         use mir::TerminatorKind::*;
         match bb_data.terminator().kind {
-            Return | Resume | Abort | GeneratorDrop | Unreachable => {}
+            Return | Resume | Terminate | GeneratorDrop | Unreachable => {}
 
             Goto { target } => propagate(target, exit_state),
 
-            Assert { target, cleanup: unwind, expected: _, msg: _, cond: _ }
+            Assert { target, unwind, expected: _, msg: _, cond: _ }
             | Drop { target, unwind, place: _ }
-            | DropAndReplace { target, unwind, value: _, place: _ }
             | FalseUnwind { real_target: target, unwind } => {
-                if let Some(unwind) = unwind {
+                if let UnwindAction::Cleanup(unwind) = unwind {
                     propagate(unwind, exit_state);
                 }
 
@@ -504,7 +503,7 @@ impl Direction for Forward {
             }
 
             Call {
-                cleanup,
+                unwind,
                 destination,
                 target,
                 func: _,
@@ -512,7 +511,7 @@ impl Direction for Forward {
                 from_hir_call: _,
                 fn_span: _,
             } => {
-                if let Some(unwind) = cleanup {
+                if let UnwindAction::Cleanup(unwind) = unwind {
                     propagate(unwind, exit_state);
                 }
 
@@ -534,9 +533,9 @@ impl Direction for Forward {
                 options: _,
                 line_spans: _,
                 destination,
-                cleanup,
+                unwind,
             } => {
-                if let Some(unwind) = cleanup {
+                if let UnwindAction::Cleanup(unwind) = unwind {
                     propagate(unwind, exit_state);
                 }
 
diff --git a/compiler/rustc_mir_dataflow/src/framework/engine.rs b/compiler/rustc_mir_dataflow/src/framework/engine.rs
index 91c3bf0ad..2abdee064 100644
--- a/compiler/rustc_mir_dataflow/src/framework/engine.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/engine.rs
@@ -15,6 +15,7 @@ use rustc_hir::def_id::DefId;
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_middle::mir::{self, traversal, BasicBlock};
 use rustc_middle::mir::{create_dump_file, dump_enabled};
+use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::symbol::{sym, Symbol};
 
@@ -285,7 +286,7 @@ where
     if tcx.sess.opts.unstable_opts.graphviz_dark_mode {
         render_opts.push(dot::RenderOption::DarkTheme);
     }
-    dot::render_opts(&graphviz, &mut buf, &render_opts)?;
+    with_no_trimmed_paths!(dot::render_opts(&graphviz, &mut buf, &render_opts)?);
 
     file.write_all(&buf)?;
 
diff --git a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs
index 96c42894b..707729f8f 100644
--- a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs
@@ -6,6 +6,7 @@ use std::{io, ops, str};
 
 use regex::Regex;
 use rustc_graphviz as dot;
+use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::graphviz_safe_def_name;
 use rustc_middle::mir::{self, BasicBlock, Body, Location};
 
@@ -34,6 +35,7 @@ where
     body: &'a Body<'tcx>,
     results: &'a Results<'tcx, A>,
     style: OutputStyle,
+    reachable: BitSet<BasicBlock>,
 }
 
 impl<'a, 'tcx, A> Formatter<'a, 'tcx, A>
@@ -41,7 +43,8 @@ where
     A: Analysis<'tcx>,
 {
     pub fn new(body: &'a Body<'tcx>, results: &'a Results<'tcx, A>, style: OutputStyle) -> Self {
-        Formatter { body, results, style }
+        let reachable = mir::traversal::reachable_as_bitset(body);
+        Formatter { body, results, style, reachable }
     }
 }
 
@@ -108,7 +111,12 @@ where
     type Edge = CfgEdge;
 
     fn nodes(&self) -> dot::Nodes<'_, Self::Node> {
-        self.body.basic_blocks.indices().collect::<Vec<_>>().into()
+        self.body
+            .basic_blocks
+            .indices()
+            .filter(|&idx| self.reachable.contains(idx))
+            .collect::<Vec<_>>()
+            .into()
     }
 
     fn edges(&self) -> dot::Edges<'_, Self::Edge> {
@@ -386,8 +394,8 @@ where
     ) -> io::Result<()> {
         let diffs = StateDiffCollector::run(body, block, self.results.results(), self.style);
 
-        let mut befores = diffs.before.map(|v| v.into_iter());
-        let mut afters = diffs.after.into_iter();
+        let mut diffs_before = diffs.before.map(|v| v.into_iter());
+        let mut diffs_after = diffs.after.into_iter();
 
         let next_in_dataflow_order = |it: &mut std::vec::IntoIter<_>| {
             if A::Direction::IS_FORWARD { it.next().unwrap() } else { it.next_back().unwrap() }
@@ -397,8 +405,8 @@ where
             let statement_str = format!("{statement:?}");
             let index_str = format!("{i}");
 
-            let after = next_in_dataflow_order(&mut afters);
-            let before = befores.as_mut().map(next_in_dataflow_order);
+            let after = next_in_dataflow_order(&mut diffs_after);
+            let before = diffs_before.as_mut().map(next_in_dataflow_order);
 
             self.write_row(w, &index_str, &statement_str, |_this, w, fmt| {
                 if let Some(before) = before {
@@ -409,11 +417,11 @@ where
             })?;
         }
 
-        let after = next_in_dataflow_order(&mut afters);
-        let before = befores.as_mut().map(next_in_dataflow_order);
+        let after = next_in_dataflow_order(&mut diffs_after);
+        let before = diffs_before.as_mut().map(next_in_dataflow_order);
 
-        assert!(afters.is_empty());
-        assert!(befores.as_ref().map_or(true, ExactSizeIterator::is_empty));
+        assert!(diffs_after.is_empty());
+        assert!(diffs_before.as_ref().map_or(true, ExactSizeIterator::is_empty));
 
         let terminator = body[block].terminator();
         let mut terminator_str = String::new();
diff --git a/compiler/rustc_mir_dataflow/src/framework/tests.rs b/compiler/rustc_mir_dataflow/src/framework/tests.rs
index 17102454a..60679b17d 100644
--- a/compiler/rustc_mir_dataflow/src/framework/tests.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/tests.rs
@@ -39,7 +39,7 @@ fn mock_body<'tcx>() -> mir::Body<'tcx> {
             args: vec![],
             destination: dummy_place.clone(),
             target: Some(mir::START_BLOCK),
-            cleanup: None,
+            unwind: mir::UnwindAction::Continue,
             from_hir_call: false,
             fn_span: DUMMY_SP,
         },
@@ -53,7 +53,7 @@ fn mock_body<'tcx>() -> mir::Body<'tcx> {
             args: vec![],
             destination: dummy_place.clone(),
             target: Some(mir::START_BLOCK),
-            cleanup: None,
+            unwind: mir::UnwindAction::Continue,
             from_hir_call: false,
             fn_span: DUMMY_SP,
         },
diff --git a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
index 6f4e7fd46..92d30f254 100644
--- a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
@@ -111,8 +111,7 @@ where
         self.super_terminator(terminator, location);
 
         match terminator.kind {
-            mir::TerminatorKind::Drop { place: dropped_place, .. }
-            | mir::TerminatorKind::DropAndReplace { place: dropped_place, .. } => {
+            mir::TerminatorKind::Drop { place: dropped_place, .. } => {
                 // Drop terminators may call custom drop glue (`Drop::drop`), which takes `&mut
                 // self` as a parameter. In the general case, a drop impl could launder that
                 // reference into the surrounding environment through a raw pointer, thus creating
@@ -126,7 +125,7 @@ where
                 }
             }
 
-            TerminatorKind::Abort
+            TerminatorKind::Terminate
             | TerminatorKind::Assert { .. }
             | TerminatorKind::Call { .. }
             | TerminatorKind::FalseEdge { .. }
diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs
index 633a5674f..1309ea525 100644
--- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs
@@ -198,8 +198,7 @@ impl DefUse {
                 | NonMutatingUseContext::Inspect
                 | NonMutatingUseContext::Move
                 | NonMutatingUseContext::ShallowBorrow
-                | NonMutatingUseContext::SharedBorrow
-                | NonMutatingUseContext::UniqueBorrow,
+                | NonMutatingUseContext::SharedBorrow,
             ) => Some(DefUse::Use),
 
             PlaceContext::MutatingUse(MutatingUseContext::Projection)
@@ -263,6 +262,7 @@ impl<'a, 'tcx> Analysis<'tcx> for MaybeTransitiveLiveLocals<'a> {
             | StatementKind::StorageDead(_)
             | StatementKind::Retag(..)
             | StatementKind::AscribeUserType(..)
+            | StatementKind::PlaceMention(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
             | StatementKind::ConstEvalCounter
diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
index fcf0ce9d8..4a5d9d520 100644
--- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
@@ -139,6 +139,7 @@ impl<'mir, 'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir, 'tc
             // Nothing to do for these. Match exhaustively so this fails to compile when new
             // variants are added.
             StatementKind::AscribeUserType(..)
+            | StatementKind::PlaceMention(..)
             | StatementKind::Coverage(..)
             | StatementKind::FakeRead(..)
             | StatementKind::ConstEvalCounter
@@ -199,10 +200,9 @@ impl<'mir, 'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir, 'tc
 
             // Nothing to do for these. Match exhaustively so this fails to compile when new
             // variants are added.
-            TerminatorKind::Abort
+            TerminatorKind::Terminate
             | TerminatorKind::Assert { .. }
             | TerminatorKind::Drop { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::FalseEdge { .. }
             | TerminatorKind::FalseUnwind { .. }
             | TerminatorKind::GeneratorDrop
@@ -237,10 +237,9 @@ impl<'mir, 'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir, 'tc
             // Nothing to do for these. Match exhaustively so this fails to compile when new
             // variants are added.
             TerminatorKind::Yield { .. }
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Assert { .. }
             | TerminatorKind::Drop { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::FalseEdge { .. }
             | TerminatorKind::FalseUnwind { .. }
             | TerminatorKind::GeneratorDrop
diff --git a/compiler/rustc_mir_dataflow/src/lib.rs b/compiler/rustc_mir_dataflow/src/lib.rs
index b1e03faff..43caa2ea9 100644
--- a/compiler/rustc_mir_dataflow/src/lib.rs
+++ b/compiler/rustc_mir_dataflow/src/lib.rs
@@ -3,7 +3,6 @@
 #![feature(exact_size_is_empty)]
 #![feature(let_chains)]
 #![feature(min_specialization)]
-#![feature(once_cell)]
 #![feature(stmt_expr_attributes)]
 #![feature(trusted_step)]
 #![recursion_limit = "256"]
@@ -46,7 +45,7 @@ pub mod storage;
 pub mod un_derefer;
 pub mod value_analysis;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub(crate) mod indexes {
     pub(crate) use super::move_paths::MovePathIndex;
diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
index 4a163028f..64ed7a29f 100644
--- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
+++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
@@ -329,6 +329,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             }
             StatementKind::Retag { .. }
             | StatementKind::AscribeUserType(..)
+            | StatementKind::PlaceMention(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
             | StatementKind::ConstEvalCounter
@@ -374,7 +375,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             // need recording.
             | TerminatorKind::Return
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::Unreachable
             | TerminatorKind::Drop { .. } => {}
@@ -392,17 +393,12 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
                 self.create_move_path(place);
                 self.gather_init(place.as_ref(), InitKind::Deep);
             }
-            TerminatorKind::DropAndReplace { place, ref value, .. } => {
-                self.create_move_path(place);
-                self.gather_operand(value);
-                self.gather_init(place.as_ref(), InitKind::Deep);
-            }
             TerminatorKind::Call {
                 ref func,
                 ref args,
                 destination,
                 target,
-                cleanup: _,
+                unwind: _,
                 from_hir_call: _,
                 fn_span: _,
             } => {
@@ -421,7 +417,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
                 options: _,
                 line_spans: _,
                 destination: _,
-                cleanup: _,
+                unwind: _,
             } => {
                 for op in operands {
                     match *op {
diff --git a/compiler/rustc_mir_dataflow/src/move_paths/mod.rs b/compiler/rustc_mir_dataflow/src/move_paths/mod.rs
index 5f22a418d..257a42cdd 100644
--- a/compiler/rustc_mir_dataflow/src/move_paths/mod.rs
+++ b/compiler/rustc_mir_dataflow/src/move_paths/mod.rs
@@ -1,6 +1,6 @@
 use crate::move_paths::builder::MoveDat;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::mir::*;
 use rustc_middle::ty::{ParamEnv, Ty, TyCtxt};
 use rustc_span::Span;
@@ -64,7 +64,7 @@ impl<'tcx> MovePath<'tcx> {
     /// Returns an iterator over the parents of `self`.
     pub fn parents<'a>(
         &self,
-        move_paths: &'a IndexVec<MovePathIndex, MovePath<'tcx>>,
+        move_paths: &'a IndexSlice<MovePathIndex, MovePath<'tcx>>,
     ) -> impl 'a + Iterator<Item = (MovePathIndex, &'a MovePath<'tcx>)> {
         let first = self.parent.map(|mpi| (mpi, &move_paths[mpi]));
         MovePathLinearIter {
@@ -78,7 +78,7 @@ impl<'tcx> MovePath<'tcx> {
     /// Returns an iterator over the immediate children of `self`.
     pub fn children<'a>(
         &self,
-        move_paths: &'a IndexVec<MovePathIndex, MovePath<'tcx>>,
+        move_paths: &'a IndexSlice<MovePathIndex, MovePath<'tcx>>,
     ) -> impl 'a + Iterator<Item = (MovePathIndex, &'a MovePath<'tcx>)> {
         let first = self.first_child.map(|mpi| (mpi, &move_paths[mpi]));
         MovePathLinearIter {
@@ -95,7 +95,7 @@ impl<'tcx> MovePath<'tcx> {
     /// `f` will **not** be called on `self`.
     pub fn find_descendant(
         &self,
-        move_paths: &IndexVec<MovePathIndex, MovePath<'_>>,
+        move_paths: &IndexSlice<MovePathIndex, MovePath<'_>>,
         f: impl Fn(MovePathIndex) -> bool,
     ) -> Option<MovePathIndex> {
         let mut todo = if let Some(child) = self.first_child {
diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs
index 401db890a..98bebc9b1 100644
--- a/compiler/rustc_mir_dataflow/src/value_analysis.rs
+++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs
@@ -36,11 +36,11 @@ use std::fmt::{Debug, Formatter};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_index::bit_set::BitSet;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 
 use crate::lattice::{HasBottom, HasTop};
 use crate::{
@@ -86,6 +86,7 @@ pub trait ValueAnalysis<'tcx> {
             StatementKind::ConstEvalCounter
             | StatementKind::Nop
             | StatementKind::FakeRead(..)
+            | StatementKind::PlaceMention(..)
             | StatementKind::Coverage(..)
             | StatementKind::AscribeUserType(..) => (),
         }
@@ -230,14 +231,14 @@ pub trait ValueAnalysis<'tcx> {
             TerminatorKind::Drop { place, .. } => {
                 state.flood_with(place.as_ref(), self.map(), Self::Value::bottom());
             }
-            TerminatorKind::DropAndReplace { .. } | TerminatorKind::Yield { .. } => {
+            TerminatorKind::Yield { .. } => {
                 // They would have an effect, but are not allowed in this phase.
                 bug!("encountered disallowed terminator");
             }
             TerminatorKind::Goto { .. }
             | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::Assert { .. }
@@ -690,7 +691,7 @@ impl Map {
         }
 
         // Recurse with all fields of this place.
-        iter_fields(ty, tcx, |variant, field, ty| {
+        iter_fields(ty, tcx, ty::ParamEnv::reveal_all(), |variant, field, ty| {
             if let Some(variant) = variant {
                 projection.push(PlaceElem::Downcast(None, variant));
                 let _ = self.make_place(local, projection);
@@ -918,7 +919,7 @@ impl<V: HasTop> ValueOrPlace<V> {
 /// Although only field projections are currently allowed, this could change in the future.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum TrackElem {
-    Field(Field),
+    Field(FieldIdx),
     Variant(VariantIdx),
     Discriminant,
 }
@@ -939,7 +940,8 @@ impl<V, T> TryFrom<ProjectionElem<V, T>> for TrackElem {
 pub fn iter_fields<'tcx>(
     ty: Ty<'tcx>,
     tcx: TyCtxt<'tcx>,
-    mut f: impl FnMut(Option<VariantIdx>, Field, Ty<'tcx>),
+    param_env: ty::ParamEnv<'tcx>,
+    mut f: impl FnMut(Option<VariantIdx>, FieldIdx, Ty<'tcx>),
 ) {
     match ty.kind() {
         ty::Tuple(list) => {
@@ -956,14 +958,14 @@ pub fn iter_fields<'tcx>(
                 for (f_index, f_def) in v_def.fields.iter().enumerate() {
                     let field_ty = f_def.ty(tcx, substs);
                     let field_ty = tcx
-                        .try_normalize_erasing_regions(ty::ParamEnv::reveal_all(), field_ty)
-                        .unwrap_or(field_ty);
+                        .try_normalize_erasing_regions(param_env, field_ty)
+                        .unwrap_or_else(|_| tcx.erase_regions(field_ty));
                     f(variant, f_index.into(), field_ty);
                 }
             }
         }
         ty::Closure(_, substs) => {
-            iter_fields(substs.as_closure().tupled_upvars_ty(), tcx, f);
+            iter_fields(substs.as_closure().tupled_upvars_ty(), tcx, param_env, f);
         }
         _ => (),
     }
@@ -1026,8 +1028,8 @@ where
 fn debug_with_context_rec<V: Debug + Eq>(
     place: PlaceIndex,
     place_str: &str,
-    new: &IndexVec<ValueIndex, V>,
-    old: Option<&IndexVec<ValueIndex, V>>,
+    new: &IndexSlice<ValueIndex, V>,
+    old: Option<&IndexSlice<ValueIndex, V>>,
     map: &Map,
     f: &mut Formatter<'_>,
 ) -> std::fmt::Result {
@@ -1067,8 +1069,8 @@ fn debug_with_context_rec<V: Debug + Eq>(
 }
 
 fn debug_with_context<V: Debug + Eq>(
-    new: &IndexVec<ValueIndex, V>,
-    old: Option<&IndexVec<ValueIndex, V>>,
+    new: &IndexSlice<ValueIndex, V>,
+    old: Option<&IndexSlice<ValueIndex, V>>,
     map: &Map,
     f: &mut Formatter<'_>,
 ) -> std::fmt::Result {
diff --git a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
index 9b4b72070..5aed89139 100644
--- a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
+++ b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
@@ -34,11 +34,6 @@ impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls {
             return;
         }
 
-        // This pass only runs on functions which themselves cannot unwind,
-        // forcibly changing the body of the function to structurally provide
-        // this guarantee by aborting on an unwind. If this function can unwind,
-        // then there's nothing to do because it already should work correctly.
-        //
         // Here we test for this function itself whether its ABI allows
         // unwinding or not.
         let body_ty = tcx.type_of(def_id).skip_binder();
@@ -74,7 +69,7 @@ impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls {
                     };
                     layout::fn_can_unwind(tcx, fn_def_id, sig.abi())
                 }
-                TerminatorKind::Drop { .. } | TerminatorKind::DropAndReplace { .. } => {
+                TerminatorKind::Drop { .. } => {
                     tcx.sess.opts.unstable_opts.panic_in_drop == PanicStrategy::Unwind
                         && layout::fn_can_unwind(tcx, None, Abi::Rust)
                 }
@@ -107,31 +102,14 @@ impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls {
             }
         }
 
-        // For call instructions which need to be terminated, we insert a
-        // singular basic block which simply terminates, and then configure the
-        // `cleanup` attribute for all calls we found to this basic block we
-        // insert which means that any unwinding that happens in the functions
-        // will force an abort of the process.
-        if !calls_to_terminate.is_empty() {
-            let bb = BasicBlockData {
-                statements: Vec::new(),
-                is_cleanup: true,
-                terminator: Some(Terminator {
-                    source_info: SourceInfo::outermost(body.span),
-                    kind: TerminatorKind::Abort,
-                }),
-            };
-            let abort_bb = body.basic_blocks_mut().push(bb);
-
-            for bb in calls_to_terminate {
-                let cleanup = body.basic_blocks_mut()[bb].terminator_mut().unwind_mut().unwrap();
-                *cleanup = Some(abort_bb);
-            }
+        for id in calls_to_terminate {
+            let cleanup = body.basic_blocks_mut()[id].terminator_mut().unwind_mut().unwrap();
+            *cleanup = UnwindAction::Terminate;
         }
 
         for id in cleanups_to_remove {
             let cleanup = body.basic_blocks_mut()[id].terminator_mut().unwind_mut().unwrap();
-            *cleanup = None;
+            *cleanup = UnwindAction::Unreachable;
         }
 
         // We may have invalidated some `cleanup` blocks so clean those up now.
diff --git a/compiler/rustc_mir_transform/src/add_call_guards.rs b/compiler/rustc_mir_transform/src/add_call_guards.rs
index 30966d22e..e1e354efa 100644
--- a/compiler/rustc_mir_transform/src/add_call_guards.rs
+++ b/compiler/rustc_mir_transform/src/add_call_guards.rs
@@ -50,10 +50,11 @@ impl AddCallGuards {
         for block in body.basic_blocks_mut() {
             match block.terminator {
                 Some(Terminator {
-                    kind: TerminatorKind::Call { target: Some(ref mut destination), cleanup, .. },
+                    kind: TerminatorKind::Call { target: Some(ref mut destination), unwind, .. },
                     source_info,
                 }) if pred_count[*destination] > 1
-                    && (cleanup.is_some() || self == &AllCallEdges) =>
+                    && (matches!(unwind, UnwindAction::Cleanup(_) | UnwindAction::Terminate)
+                        || self == &AllCallEdges) =>
                 {
                     // It's a critical edge, break it
                     let call_guard = BasicBlockData {
diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
index 9b2260f68..896fcd9cd 100644
--- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
+++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
@@ -64,9 +64,6 @@ fn add_moves_for_packed_drops_patch<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>)
             {
                 add_move_for_packed_drop(tcx, body, &mut patch, terminator, loc, data.is_cleanup);
             }
-            TerminatorKind::DropAndReplace { .. } => {
-                span_bug!(terminator.source_info.span, "replace in AddMovesForPackedDrops");
-            }
             _ => {}
         }
     }
diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs
index 7d2146214..916f2904d 100644
--- a/compiler/rustc_mir_transform/src/add_retag.rs
+++ b/compiler/rustc_mir_transform/src/add_retag.rs
@@ -100,7 +100,7 @@ impl<'tcx> MirPass<'tcx> for AddRetag {
                     }
 
                     // `Drop` is also a call, but it doesn't return anything so we are good.
-                    TerminatorKind::Drop { .. } | TerminatorKind::DropAndReplace { .. } => None,
+                    TerminatorKind::Drop { .. } => None,
                     // Not a block ending in a Call -> ignore.
                     _ => None,
                 }
diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs
new file mode 100644
index 000000000..9311666c9
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/check_alignment.rs
@@ -0,0 +1,242 @@
+use crate::MirPass;
+use rustc_hir::def_id::DefId;
+use rustc_hir::lang_items::LangItem;
+use rustc_index::vec::IndexVec;
+use rustc_middle::mir::*;
+use rustc_middle::mir::{
+    interpret::{ConstValue, Scalar},
+    visit::{PlaceContext, Visitor},
+};
+use rustc_middle::ty::{Ty, TyCtxt, TypeAndMut};
+use rustc_session::Session;
+
+pub struct CheckAlignment;
+
+impl<'tcx> MirPass<'tcx> for CheckAlignment {
+    fn is_enabled(&self, sess: &Session) -> bool {
+        sess.opts.debug_assertions
+    }
+
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        // This pass emits new panics. If for whatever reason we do not have a panic
+        // implementation, running this pass may cause otherwise-valid code to not compile.
+        if tcx.lang_items().get(LangItem::PanicImpl).is_none() {
+            return;
+        }
+
+        let basic_blocks = body.basic_blocks.as_mut();
+        let local_decls = &mut body.local_decls;
+
+        for block in (0..basic_blocks.len()).rev() {
+            let block = block.into();
+            for statement_index in (0..basic_blocks[block].statements.len()).rev() {
+                let location = Location { block, statement_index };
+                let statement = &basic_blocks[block].statements[statement_index];
+                let source_info = statement.source_info;
+
+                let mut finder = PointerFinder {
+                    local_decls,
+                    tcx,
+                    pointers: Vec::new(),
+                    def_id: body.source.def_id(),
+                };
+                for (pointer, pointee_ty) in finder.find_pointers(statement) {
+                    debug!("Inserting alignment check for {:?}", pointer.ty(&*local_decls, tcx).ty);
+
+                    let new_block = split_block(basic_blocks, location);
+                    insert_alignment_check(
+                        tcx,
+                        local_decls,
+                        &mut basic_blocks[block],
+                        pointer,
+                        pointee_ty,
+                        source_info,
+                        new_block,
+                    );
+                }
+            }
+        }
+    }
+}
+
+impl<'tcx, 'a> PointerFinder<'tcx, 'a> {
+    fn find_pointers(&mut self, statement: &Statement<'tcx>) -> Vec<(Place<'tcx>, Ty<'tcx>)> {
+        self.pointers.clear();
+        self.visit_statement(statement, Location::START);
+        core::mem::take(&mut self.pointers)
+    }
+}
+
+struct PointerFinder<'tcx, 'a> {
+    local_decls: &'a mut LocalDecls<'tcx>,
+    tcx: TyCtxt<'tcx>,
+    def_id: DefId,
+    pointers: Vec<(Place<'tcx>, Ty<'tcx>)>,
+}
+
+impl<'tcx, 'a> Visitor<'tcx> for PointerFinder<'tcx, 'a> {
+    fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) {
+        if let Rvalue::AddressOf(..) = rvalue {
+            // Ignore dereferences inside of an AddressOf
+            return;
+        }
+        self.super_rvalue(rvalue, location);
+    }
+
+    fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) {
+        if let PlaceContext::NonUse(_) = context {
+            return;
+        }
+        if !place.is_indirect() {
+            return;
+        }
+
+        let pointer = Place::from(place.local);
+        let pointer_ty = pointer.ty(&*self.local_decls, self.tcx).ty;
+
+        // We only want to check unsafe pointers
+        if !pointer_ty.is_unsafe_ptr() {
+            trace!("Indirect, but not an unsafe ptr, not checking {:?}", pointer_ty);
+            return;
+        }
+
+        let Some(pointee) = pointer_ty.builtin_deref(true) else {
+            debug!("Indirect but no builtin deref: {:?}", pointer_ty);
+            return;
+        };
+        let mut pointee_ty = pointee.ty;
+        if pointee_ty.is_array() || pointee_ty.is_slice() || pointee_ty.is_str() {
+            pointee_ty = pointee_ty.sequence_element_type(self.tcx);
+        }
+
+        if !pointee_ty.is_sized(self.tcx, self.tcx.param_env_reveal_all_normalized(self.def_id)) {
+            debug!("Unsafe pointer, but unsized: {:?}", pointer_ty);
+            return;
+        }
+
+        if [self.tcx.types.bool, self.tcx.types.i8, self.tcx.types.u8, self.tcx.types.str_]
+            .contains(&pointee_ty)
+        {
+            debug!("Trivially aligned pointee type: {:?}", pointer_ty);
+            return;
+        }
+
+        self.pointers.push((pointer, pointee_ty))
+    }
+}
+
+fn split_block(
+    basic_blocks: &mut IndexVec<BasicBlock, BasicBlockData<'_>>,
+    location: Location,
+) -> BasicBlock {
+    let block_data = &mut basic_blocks[location.block];
+
+    // Drain every statement after this one and move the current terminator to a new basic block
+    let new_block = BasicBlockData {
+        statements: block_data.statements.split_off(location.statement_index),
+        terminator: block_data.terminator.take(),
+        is_cleanup: block_data.is_cleanup,
+    };
+
+    basic_blocks.push(new_block)
+}
+
+fn insert_alignment_check<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    local_decls: &mut IndexVec<Local, LocalDecl<'tcx>>,
+    block_data: &mut BasicBlockData<'tcx>,
+    pointer: Place<'tcx>,
+    pointee_ty: Ty<'tcx>,
+    source_info: SourceInfo,
+    new_block: BasicBlock,
+) {
+    // Cast the pointer to a *const ()
+    let const_raw_ptr = tcx.mk_ptr(TypeAndMut { ty: tcx.types.unit, mutbl: Mutability::Not });
+    let rvalue = Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(pointer), const_raw_ptr);
+    let thin_ptr = local_decls.push(LocalDecl::with_source_info(const_raw_ptr, source_info)).into();
+    block_data
+        .statements
+        .push(Statement { source_info, kind: StatementKind::Assign(Box::new((thin_ptr, rvalue))) });
+
+    // Transmute the pointer to a usize (equivalent to `ptr.addr()`)
+    let rvalue = Rvalue::Cast(CastKind::Transmute, Operand::Copy(thin_ptr), tcx.types.usize);
+    let addr = local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into();
+    block_data
+        .statements
+        .push(Statement { source_info, kind: StatementKind::Assign(Box::new((addr, rvalue))) });
+
+    // Get the alignment of the pointee
+    let alignment =
+        local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into();
+    let rvalue = Rvalue::NullaryOp(NullOp::AlignOf, pointee_ty);
+    block_data.statements.push(Statement {
+        source_info,
+        kind: StatementKind::Assign(Box::new((alignment, rvalue))),
+    });
+
+    // Subtract 1 from the alignment to get the alignment mask
+    let alignment_mask =
+        local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into();
+    let one = Operand::Constant(Box::new(Constant {
+        span: source_info.span,
+        user_ty: None,
+        literal: ConstantKind::Val(
+            ConstValue::Scalar(Scalar::from_target_usize(1, &tcx)),
+            tcx.types.usize,
+        ),
+    }));
+    block_data.statements.push(Statement {
+        source_info,
+        kind: StatementKind::Assign(Box::new((
+            alignment_mask,
+            Rvalue::BinaryOp(BinOp::Sub, Box::new((Operand::Copy(alignment), one))),
+        ))),
+    });
+
+    // BitAnd the alignment mask with the pointer
+    let alignment_bits =
+        local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into();
+    block_data.statements.push(Statement {
+        source_info,
+        kind: StatementKind::Assign(Box::new((
+            alignment_bits,
+            Rvalue::BinaryOp(
+                BinOp::BitAnd,
+                Box::new((Operand::Copy(addr), Operand::Copy(alignment_mask))),
+            ),
+        ))),
+    });
+
+    // Check if the alignment bits are all zero
+    let is_ok = local_decls.push(LocalDecl::with_source_info(tcx.types.bool, source_info)).into();
+    let zero = Operand::Constant(Box::new(Constant {
+        span: source_info.span,
+        user_ty: None,
+        literal: ConstantKind::Val(
+            ConstValue::Scalar(Scalar::from_target_usize(0, &tcx)),
+            tcx.types.usize,
+        ),
+    }));
+    block_data.statements.push(Statement {
+        source_info,
+        kind: StatementKind::Assign(Box::new((
+            is_ok,
+            Rvalue::BinaryOp(BinOp::Eq, Box::new((Operand::Copy(alignment_bits), zero.clone()))),
+        ))),
+    });
+
+    // Set this block's terminator to our assert, continuing to new_block if we pass
+    block_data.terminator = Some(Terminator {
+        source_info,
+        kind: TerminatorKind::Assert {
+            cond: Operand::Copy(is_ok),
+            expected: true,
+            target: new_block,
+            msg: AssertKind::MisalignedPointerDereference {
+                required: Operand::Copy(alignment),
+                found: Operand::Copy(addr),
+            },
+            unwind: UnwindAction::Terminate,
+        },
+    });
+}
diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
index 536745d2c..3d32c5865 100644
--- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
+++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
@@ -24,7 +24,7 @@ struct ConstMutationChecker<'a, 'tcx> {
 
 impl<'tcx> ConstMutationChecker<'_, 'tcx> {
     fn is_const_item(&self, local: Local) -> Option<DefId> {
-        if let Some(box LocalInfo::ConstRef { def_id }) = self.body.local_decls[local].local_info {
+        if let LocalInfo::ConstRef { def_id } = *self.body.local_decls[local].local_info() {
             Some(def_id)
         } else {
             None
diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs
index d00ee1f4b..d908f6b3a 100644
--- a/compiler/rustc_mir_transform/src/check_unsafety.rs
+++ b/compiler/rustc_mir_transform/src/check_unsafety.rs
@@ -1,4 +1,4 @@
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::unord::{UnordItems, UnordSet};
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
@@ -24,7 +24,7 @@ pub struct UnsafetyChecker<'a, 'tcx> {
     param_env: ty::ParamEnv<'tcx>,
 
     /// Used `unsafe` blocks in this function. This is used for the "unused_unsafe" lint.
-    used_unsafe_blocks: FxHashSet<HirId>,
+    used_unsafe_blocks: UnordSet<HirId>,
 }
 
 impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> {
@@ -55,10 +55,9 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
             | TerminatorKind::Drop { .. }
             | TerminatorKind::Yield { .. }
             | TerminatorKind::Assert { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::FalseEdge { .. }
@@ -101,13 +100,16 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
             | StatementKind::StorageLive(..)
             | StatementKind::StorageDead(..)
             | StatementKind::Retag { .. }
-            | StatementKind::AscribeUserType(..)
+            | StatementKind::PlaceMention(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
             | StatementKind::ConstEvalCounter
             | StatementKind::Nop => {
                 // safe (at least as emitted during MIR construction)
             }
+            // `AscribeUserType` just exists to help MIR borrowck.
+            // It has no semantics, and everything is already reported by `PlaceMention`.
+            StatementKind::AscribeUserType(..) => return,
         }
         self.super_statement(statement, location);
     }
@@ -129,7 +131,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
                     let def_id = def_id.expect_local();
                     let UnsafetyCheckResult { violations, used_unsafe_blocks, .. } =
                         self.tcx.unsafety_check_result(def_id);
-                    self.register_violations(violations, used_unsafe_blocks.iter().copied());
+                    self.register_violations(violations, used_unsafe_blocks.items().copied());
                 }
             },
             _ => {}
@@ -151,7 +153,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
                         let local_def_id = def_id.expect_local();
                         let UnsafetyCheckResult { violations, used_unsafe_blocks, .. } =
                             self.tcx.unsafety_check_result(local_def_id);
-                        self.register_violations(violations, used_unsafe_blocks.iter().copied());
+                        self.register_violations(violations, used_unsafe_blocks.items().copied());
                     }
                 }
             }
@@ -180,7 +182,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
             // If the projection root is an artificial local that we introduced when
             // desugaring `static`, give a more specific error message
             // (avoid the general "raw pointer" clause below, that would only be confusing).
-            if let Some(box LocalInfo::StaticRef { def_id, .. }) = decl.local_info {
+            if let LocalInfo::StaticRef { def_id, .. } = *decl.local_info() {
                 if self.tcx.is_mutable_static(def_id) {
                     self.require_unsafe(
                         UnsafetyViolationKind::General,
@@ -268,14 +270,14 @@ impl<'tcx> UnsafetyChecker<'_, 'tcx> {
             .lint_root;
         self.register_violations(
             [&UnsafetyViolation { source_info, lint_root, kind, details }],
-            [],
+            UnordItems::empty(),
         );
     }
 
     fn register_violations<'a>(
         &mut self,
         violations: impl IntoIterator<Item = &'a UnsafetyViolation>,
-        new_used_unsafe_blocks: impl IntoIterator<Item = HirId>,
+        new_used_unsafe_blocks: UnordItems<HirId, impl Iterator<Item = HirId>>,
     ) {
         let safety = self.body.source_scopes[self.source_info.scope]
             .local_data
@@ -308,9 +310,7 @@ impl<'tcx> UnsafetyChecker<'_, 'tcx> {
             }),
         };
 
-        new_used_unsafe_blocks.into_iter().for_each(|hir_id| {
-            self.used_unsafe_blocks.insert(hir_id);
-        });
+        self.used_unsafe_blocks.extend_unord(new_used_unsafe_blocks);
     }
     fn check_mut_borrowing_layout_constrained_field(
         &mut self,
@@ -407,7 +407,7 @@ enum Context {
 
 struct UnusedUnsafeVisitor<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    used_unsafe_blocks: &'a FxHashSet<HirId>,
+    used_unsafe_blocks: &'a UnordSet<HirId>,
     context: Context,
     unused_unsafes: &'a mut Vec<(HirId, UnusedUnsafe)>,
 }
@@ -458,7 +458,7 @@ impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> {
 fn check_unused_unsafe(
     tcx: TyCtxt<'_>,
     def_id: LocalDefId,
-    used_unsafe_blocks: &FxHashSet<HirId>,
+    used_unsafe_blocks: &UnordSet<HirId>,
 ) -> Vec<(HirId, UnusedUnsafe)> {
     let body_id = tcx.hir().maybe_body_owned_by(def_id);
 
@@ -505,7 +505,7 @@ fn unsafety_check_result(
     if body.is_custom_mir() {
         return tcx.arena.alloc(UnsafetyCheckResult {
             violations: Vec::new(),
-            used_unsafe_blocks: FxHashSet::default(),
+            used_unsafe_blocks: Default::default(),
             unused_unsafes: Some(Vec::new()),
         });
     }
diff --git a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
index d435d3ee6..0923824db 100644
--- a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
+++ b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
@@ -24,6 +24,7 @@ impl<'tcx> MirPass<'tcx> for CleanupPostBorrowck {
             for statement in basic_block.statements.iter_mut() {
                 match statement.kind {
                     StatementKind::AscribeUserType(..)
+                    | StatementKind::PlaceMention(..)
                     | StatementKind::Assign(box (_, Rvalue::Ref(_, BorrowKind::Shallow, _)))
                     | StatementKind::FakeRead(..) => statement.make_nop(),
                     _ => (),
diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs
index 6b2eefce2..1bb45341e 100644
--- a/compiler/rustc_mir_transform/src/const_prop.rs
+++ b/compiler/rustc_mir_transform/src/const_prop.rs
@@ -1,15 +1,13 @@
 //! Propagates constants for early reporting of statically known
 //! assertion failures
 
-use std::cell::Cell;
-
 use either::Right;
 
 use rustc_const_eval::const_eval::CheckAlignment;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::def::DefKind;
 use rustc_index::bit_set::BitSet;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{
     MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor,
 };
@@ -17,7 +15,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout};
 use rustc_middle::ty::InternalSubsts;
 use rustc_middle::ty::{self, ConstKind, Instance, ParamEnv, Ty, TyCtxt, TypeVisitableExt};
-use rustc_span::{def_id::DefId, Span};
+use rustc_span::{def_id::DefId, Span, DUMMY_SP};
 use rustc_target::abi::{self, Align, HasDataLayout, Size, TargetDataLayout};
 use rustc_target::spec::abi::Abi as CallAbi;
 use rustc_trait_selection::traits;
@@ -25,8 +23,8 @@ use rustc_trait_selection::traits;
 use crate::MirPass;
 use rustc_const_eval::interpret::{
     self, compile_time_machine, AllocId, ConstAllocation, ConstValue, CtfeValidationMode, Frame,
-    ImmTy, Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, PlaceTy,
-    Pointer, Scalar, StackPopCleanup, StackPopUnwind,
+    ImmTy, Immediate, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, PlaceTy, Pointer,
+    Scalar, StackPopCleanup,
 };
 
 /// The maximum number of bytes that we'll allocate space for a local or the return value.
@@ -56,7 +54,7 @@ pub struct ConstProp;
 
 impl<'tcx> MirPass<'tcx> for ConstProp {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
-        sess.mir_opt_level() >= 1
+        sess.mir_opt_level() >= 2
     }
 
     #[instrument(skip(self, tcx), level = "debug")]
@@ -117,10 +115,7 @@ impl<'tcx> MirPass<'tcx> for ConstProp {
             .predicates
             .iter()
             .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None });
-        if traits::impossible_predicates(
-            tcx,
-            traits::elaborate_predicates(tcx, predicates).map(|o| o.predicate).collect(),
-        ) {
+        if traits::impossible_predicates(tcx, traits::elaborate(tcx, predicates).collect()) {
             trace!("ConstProp skipped for {:?}: found unsatisfiable predicates", def_id);
             return;
         }
@@ -129,7 +124,7 @@ impl<'tcx> MirPass<'tcx> for ConstProp {
 
         let dummy_body = &Body::new(
             body.source,
-            (*body.basic_blocks).clone(),
+            (*body.basic_blocks).to_owned(),
             body.source_scopes.clone(),
             body.local_decls.clone(),
             Default::default(),
@@ -154,22 +149,15 @@ impl<'tcx> MirPass<'tcx> for ConstProp {
 pub struct ConstPropMachine<'mir, 'tcx> {
     /// The virtual call stack.
     stack: Vec<Frame<'mir, 'tcx>>,
-    /// `OnlyInsideOwnBlock` locals that were written in the current block get erased at the end.
     pub written_only_inside_own_block_locals: FxHashSet<Local>,
-    /// Locals that need to be cleared after every block terminates.
-    pub only_propagate_inside_block_locals: BitSet<Local>,
     pub can_const_prop: IndexVec<Local, ConstPropMode>,
 }
 
 impl ConstPropMachine<'_, '_> {
-    pub fn new(
-        only_propagate_inside_block_locals: BitSet<Local>,
-        can_const_prop: IndexVec<Local, ConstPropMode>,
-    ) -> Self {
+    pub fn new(can_const_prop: IndexVec<Local, ConstPropMode>) -> Self {
         Self {
             stack: Vec::new(),
             written_only_inside_own_block_locals: Default::default(),
-            only_propagate_inside_block_locals,
             can_const_prop,
         }
     }
@@ -189,7 +177,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
     }
 
     #[inline(always)]
-    fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool {
+    fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>, _layout: TyAndLayout<'tcx>) -> bool {
         false // for now, we don't enforce validity
     }
     fn alignment_check_failed(
@@ -218,7 +206,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
         _args: &[OpTy<'tcx>],
         _destination: &PlaceTy<'tcx>,
         _target: Option<BasicBlock>,
-        _unwind: StackPopUnwind,
+        _unwind: UnwindAction,
     ) -> InterpResult<'tcx, Option<(&'mir Body<'tcx>, ty::Instance<'tcx>)>> {
         Ok(None)
     }
@@ -229,7 +217,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
         _args: &[OpTy<'tcx>],
         _destination: &PlaceTy<'tcx>,
         _target: Option<BasicBlock>,
-        _unwind: StackPopUnwind,
+        _unwind: UnwindAction,
     ) -> InterpResult<'tcx> {
         throw_machine_stop_str!("calling intrinsics isn't supported in ConstProp")
     }
@@ -237,7 +225,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
     fn assert_panic(
         _ecx: &mut InterpCx<'mir, 'tcx, Self>,
         _msg: &rustc_middle::mir::AssertMessage<'tcx>,
-        _unwind: Option<rustc_middle::mir::BasicBlock>,
+        _unwind: rustc_middle::mir::UnwindAction,
     ) -> InterpResult<'tcx> {
         bug!("panics terminators are not evaluated in ConstProp")
     }
@@ -257,16 +245,17 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
         frame: usize,
         local: Local,
     ) -> InterpResult<'tcx, &'a mut interpret::Operand<Self::Provenance>> {
-        if ecx.machine.can_const_prop[local] == ConstPropMode::NoPropagation {
-            throw_machine_stop_str!("tried to write to a local that is marked as not propagatable")
-        }
-        if frame == 0 && ecx.machine.only_propagate_inside_block_locals.contains(local) {
-            trace!(
-                "mutating local {:?} which is restricted to its block. \
-                Will remove it from const-prop after block is finished.",
-                local
-            );
-            ecx.machine.written_only_inside_own_block_locals.insert(local);
+        assert_eq!(frame, 0);
+        match ecx.machine.can_const_prop[local] {
+            ConstPropMode::NoPropagation => {
+                throw_machine_stop_str!(
+                    "tried to write to a local that is marked as not propagatable"
+                )
+            }
+            ConstPropMode::OnlyInsideOwnBlock => {
+                ecx.machine.written_only_inside_own_block_locals.insert(local);
+            }
+            ConstPropMode::FullConstProp => {}
         }
         ecx.machine.stack[frame].locals[local].access_mut()
     }
@@ -327,10 +316,7 @@ struct ConstPropagator<'mir, 'tcx> {
     ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>,
     tcx: TyCtxt<'tcx>,
     param_env: ParamEnv<'tcx>,
-    local_decls: &'mir IndexVec<Local, LocalDecl<'tcx>>,
-    // Because we have `MutVisitor` we can't obtain the `SourceInfo` from a `Location`. So we store
-    // the last known `SourceInfo` here and just keep revisiting it.
-    source_info: Option<SourceInfo>,
+    local_decls: &'mir IndexSlice<Local, LocalDecl<'tcx>>,
 }
 
 impl<'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'_, 'tcx> {
@@ -374,17 +360,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         let param_env = tcx.param_env_reveal_all_normalized(def_id);
 
         let can_const_prop = CanConstProp::check(tcx, param_env, body);
-        let mut only_propagate_inside_block_locals = BitSet::new_empty(can_const_prop.len());
-        for (l, mode) in can_const_prop.iter_enumerated() {
-            if *mode == ConstPropMode::OnlyInsideOwnBlock {
-                only_propagate_inside_block_locals.insert(l);
-            }
-        }
         let mut ecx = InterpCx::new(
             tcx,
             tcx.def_span(def_id),
             param_env,
-            ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop),
+            ConstPropMachine::new(can_const_prop),
         );
 
         let ret_layout = ecx
@@ -411,13 +391,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         )
         .expect("failed to push initial stack frame");
 
-        ConstPropagator {
-            ecx,
-            tcx,
-            param_env,
-            local_decls: &dummy_body.local_decls,
-            source_info: None,
-        }
+        ConstPropagator { ecx, tcx, param_env, local_decls: &dummy_body.local_decls }
     }
 
     fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
@@ -446,10 +420,9 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
     /// Remove `local` from the pool of `Locals`. Allows writing to them,
     /// but not reading from them anymore.
     fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) {
-        ecx.frame_mut().locals[local] = LocalState {
-            value: LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)),
-            layout: Cell::new(None),
-        };
+        ecx.frame_mut().locals[local].value =
+            LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit));
+        ecx.machine.written_only_inside_own_block_locals.remove(&local);
     }
 
     /// Returns the value, if any, of evaluating `c`.
@@ -492,11 +465,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                         scalar,
                     )) = *value
                     {
-                        *operand = self.operand_from_scalar(
-                            scalar,
-                            value.layout.ty,
-                            self.source_info.unwrap().span,
-                        );
+                        *operand = self.operand_from_scalar(scalar, value.layout.ty);
                     }
                 }
             }
@@ -504,7 +473,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         }
     }
 
-    fn const_prop(&mut self, rvalue: &Rvalue<'tcx>, place: Place<'tcx>) -> Option<()> {
+    fn check_rvalue(&mut self, rvalue: &Rvalue<'tcx>) -> Option<()> {
         // Perform any special handling for specific Rvalue types.
         // Generally, checks here fall into one of two categories:
         //   1. Additional checking to provide useful lints to the user
@@ -532,6 +501,15 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
 
                 return None;
             }
+            // Do not try creating references, nor any types with potentially-complex
+            // invariants. This avoids an issue where checking validity would do a
+            // bunch of work generating a nice message about the invariant violation,
+            // only to not show it to anyone (since this isn't the lint).
+            Rvalue::Cast(CastKind::Transmute, op, dst_ty) if !dst_ty.is_primitive() => {
+                trace!("skipping Transmute of {:?} to {:?}", op, dst_ty);
+
+                return None;
+            }
 
             // There's no other checking to do at this time.
             Rvalue::Aggregate(..)
@@ -561,7 +539,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             return None;
         }
 
-        self.eval_rvalue_with_identities(rvalue, place)
+        Some(())
     }
 
     // Attempt to use algebraic identities to eliminate constant expressions
@@ -621,20 +599,24 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
     }
 
     /// Creates a new `Operand::Constant` from a `Scalar` value
-    fn operand_from_scalar(&self, scalar: Scalar, ty: Ty<'tcx>, span: Span) -> Operand<'tcx> {
+    fn operand_from_scalar(&self, scalar: Scalar, ty: Ty<'tcx>) -> Operand<'tcx> {
         Operand::Constant(Box::new(Constant {
-            span,
+            span: DUMMY_SP,
             user_ty: None,
             literal: ConstantKind::from_scalar(self.tcx, scalar, ty),
         }))
     }
 
-    fn replace_with_const(
-        &mut self,
-        rval: &mut Rvalue<'tcx>,
-        value: &OpTy<'tcx>,
-        source_info: SourceInfo,
-    ) {
+    fn replace_with_const(&mut self, place: Place<'tcx>, rval: &mut Rvalue<'tcx>) {
+        // This will return None if the above `const_prop` invocation only "wrote" a
+        // type whose creation requires no write. E.g. a generator whose initial state
+        // consists solely of uninitialized memory (so it doesn't capture any locals).
+        let Some(ref value) = self.get_const(place) else { return };
+        if !self.should_const_prop(value) {
+            return;
+        }
+        trace!("replacing {:?}={:?} with {:?}", place, rval, value);
+
         if let Rvalue::Use(Operand::Constant(c)) = rval {
             match c.literal {
                 ConstantKind::Ty(c) if matches!(c.kind(), ConstKind::Unevaluated(..)) => {}
@@ -664,11 +646,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         if let Some(Right(imm)) = imm {
             match *imm {
                 interpret::Immediate::Scalar(scalar) => {
-                    *rval = Rvalue::Use(self.operand_from_scalar(
-                        scalar,
-                        value.layout.ty,
-                        source_info.span,
-                    ));
+                    *rval = Rvalue::Use(self.operand_from_scalar(scalar, value.layout.ty));
                 }
                 Immediate::ScalarPair(..) => {
                     // Found a value represented as a pair. For now only do const-prop if the type
@@ -701,7 +679,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                                 let const_val = ConstValue::ByRef { alloc, offset: Size::ZERO };
                                 let literal = ConstantKind::Val(const_val, ty);
                                 *rval = Rvalue::Use(Operand::Constant(Box::new(Constant {
-                                    span: source_info.span,
+                                    span: DUMMY_SP,
                                     user_ty: None,
                                     literal,
                                 })));
@@ -730,6 +708,19 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             _ => false,
         }
     }
+
+    fn ensure_not_propagated(&self, local: Local) {
+        if cfg!(debug_assertions) {
+            assert!(
+                self.get_const(local.into()).is_none()
+                    || self
+                        .layout_of(self.local_decls[local].ty)
+                        .map_or(true, |layout| layout.is_zst()),
+                "failed to remove values for `{local:?}`, value={:?}",
+                self.get_const(local.into()),
+            )
+        }
+    }
 }
 
 /// The mode that `ConstProp` is allowed to run in for a given `Local`.
@@ -739,8 +730,6 @@ pub enum ConstPropMode {
     FullConstProp,
     /// The `Local` can only be propagated into and from its own block.
     OnlyInsideOwnBlock,
-    /// The `Local` can be propagated into but reads cannot be propagated.
-    OnlyPropagateInto,
     /// The `Local` cannot be part of propagation at all. Any statement
     /// referencing it either for reading or writing will not get propagated.
     NoPropagation,
@@ -750,8 +739,6 @@ pub struct CanConstProp {
     can_const_prop: IndexVec<Local, ConstPropMode>,
     // False at the beginning. Once set, no more assignments are allowed to that local.
     found_assignment: BitSet<Local>,
-    // Cache of locals' information
-    local_kinds: IndexVec<Local, LocalKind>,
 }
 
 impl CanConstProp {
@@ -764,10 +751,6 @@ impl CanConstProp {
         let mut cpv = CanConstProp {
             can_const_prop: IndexVec::from_elem(ConstPropMode::FullConstProp, &body.local_decls),
             found_assignment: BitSet::new_empty(body.local_decls.len()),
-            local_kinds: IndexVec::from_fn_n(
-                |local| body.local_kind(local),
-                body.local_decls.len(),
-            ),
         };
         for (local, val) in cpv.can_const_prop.iter_enumerated_mut() {
             let ty = body.local_decls[local].ty;
@@ -780,24 +763,10 @@ impl CanConstProp {
                     continue;
                 }
             }
-            // Cannot use args at all
-            // Cannot use locals because if x < y { y - x } else { x - y } would
-            //        lint for x != y
-            // FIXME(oli-obk): lint variables until they are used in a condition
-            // FIXME(oli-obk): lint if return value is constant
-            if cpv.local_kinds[local] == LocalKind::Arg {
-                *val = ConstPropMode::OnlyPropagateInto;
-                trace!(
-                    "local {:?} can't be const propagated because it's a function argument",
-                    local
-                );
-            } else if cpv.local_kinds[local] == LocalKind::Var {
-                *val = ConstPropMode::OnlyInsideOwnBlock;
-                trace!(
-                    "local {:?} will only be propagated inside its block, because it's a user variable",
-                    local
-                );
-            }
+        }
+        // Consider that arguments are assigned on entry.
+        for arg in body.args_iter() {
+            cpv.found_assignment.insert(arg);
         }
         cpv.visit_body(&body);
         cpv.can_const_prop
@@ -827,7 +796,6 @@ impl Visitor<'_> for CanConstProp {
                         // states as applicable.
                         ConstPropMode::OnlyInsideOwnBlock => {}
                         ConstPropMode::NoPropagation => {}
-                        ConstPropMode::OnlyPropagateInto => {}
                         other @ ConstPropMode::FullConstProp => {
                             trace!(
                                 "local {:?} can't be propagated because of multiple assignments. Previous state: {:?}",
@@ -854,7 +822,6 @@ impl Visitor<'_> for CanConstProp {
             // mutation.
             | NonMutatingUse(NonMutatingUseContext::SharedBorrow)
             | NonMutatingUse(NonMutatingUseContext::ShallowBorrow)
-            | NonMutatingUse(NonMutatingUseContext::UniqueBorrow)
             | NonMutatingUse(NonMutatingUseContext::AddressOf)
             | MutatingUse(MutatingUseContext::Borrow)
             | MutatingUse(MutatingUseContext::AddressOf) => {
@@ -886,48 +853,23 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
         }
     }
 
-    fn visit_constant(&mut self, constant: &mut Constant<'tcx>, location: Location) {
-        trace!("visit_constant: {:?}", constant);
-        self.super_constant(constant, location);
-        self.eval_constant(constant);
-    }
+    fn visit_assign(
+        &mut self,
+        place: &mut Place<'tcx>,
+        rvalue: &mut Rvalue<'tcx>,
+        location: Location,
+    ) {
+        self.super_assign(place, rvalue, location);
 
-    fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) {
-        trace!("visit_statement: {:?}", statement);
-        let source_info = statement.source_info;
-        self.source_info = Some(source_info);
-        match statement.kind {
-            StatementKind::Assign(box (place, ref mut rval)) => {
-                let can_const_prop = self.ecx.machine.can_const_prop[place.local];
-                if let Some(()) = self.const_prop(rval, place) {
-                    // This will return None if the above `const_prop` invocation only "wrote" a
-                    // type whose creation requires no write. E.g. a generator whose initial state
-                    // consists solely of uninitialized memory (so it doesn't capture any locals).
-                    if let Some(ref value) = self.get_const(place) && self.should_const_prop(value) {
-                        trace!("replacing {:?} with {:?}", rval, value);
-                        self.replace_with_const(rval, value, source_info);
-                        if can_const_prop == ConstPropMode::FullConstProp
-                            || can_const_prop == ConstPropMode::OnlyInsideOwnBlock
-                        {
-                            trace!("propagated into {:?}", place);
-                        }
-                    }
-                    match can_const_prop {
-                        ConstPropMode::OnlyInsideOwnBlock => {
-                            trace!(
-                                "found local restricted to its block. \
-                                Will remove it from const-prop after block is finished. Local: {:?}",
-                                place.local
-                            );
-                        }
-                        ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => {
-                            trace!("can't propagate into {:?}", place);
-                            if place.local != RETURN_PLACE {
-                                Self::remove_const(&mut self.ecx, place.local);
-                            }
-                        }
-                        ConstPropMode::FullConstProp => {}
-                    }
+        let Some(()) = self.check_rvalue(rvalue) else { return };
+
+        match self.ecx.machine.can_const_prop[place.local] {
+            // Do nothing if the place is indirect.
+            _ if place.is_indirect() => {}
+            ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local),
+            ConstPropMode::OnlyInsideOwnBlock | ConstPropMode::FullConstProp => {
+                if let Some(()) = self.eval_rvalue_with_identities(rvalue, *place) {
+                    self.replace_with_const(*place, rvalue);
                 } else {
                     // Const prop failed, so erase the destination, ensuring that whatever happens
                     // from here on, does not know about the previous value.
@@ -947,8 +889,22 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
                     Self::remove_const(&mut self.ecx, place.local);
                 }
             }
+        }
+    }
+
+    fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) {
+        trace!("visit_statement: {:?}", statement);
+
+        // We want to evaluate operands before any change to the assigned-to value,
+        // so we recurse first.
+        self.super_statement(statement, location);
+
+        match statement.kind {
             StatementKind::SetDiscriminant { ref place, .. } => {
                 match self.ecx.machine.can_const_prop[place.local] {
+                    // Do nothing if the place is indirect.
+                    _ if place.is_indirect() => {}
+                    ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local),
                     ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => {
                         if self.ecx.statement(statement).is_ok() {
                             trace!("propped discriminant into {:?}", place);
@@ -956,28 +912,22 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
                             Self::remove_const(&mut self.ecx, place.local);
                         }
                     }
-                    ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => {
-                        Self::remove_const(&mut self.ecx, place.local);
-                    }
                 }
             }
-            StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => {
+            StatementKind::StorageLive(local) => {
                 let frame = self.ecx.frame_mut();
-                frame.locals[local].value = if let StatementKind::StorageLive(_) = statement.kind {
-                    LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit))
-                } else {
-                    LocalValue::Dead
-                };
+                frame.locals[local].value =
+                    LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit));
+            }
+            StatementKind::StorageDead(local) => {
+                let frame = self.ecx.frame_mut();
+                frame.locals[local].value = LocalValue::Dead;
             }
             _ => {}
         }
-
-        self.super_statement(statement, location);
     }
 
     fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) {
-        let source_info = terminator.source_info;
-        self.source_info = Some(source_info);
         self.super_terminator(terminator, location);
 
         match &mut terminator.kind {
@@ -987,11 +937,7 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
                     && self.should_const_prop(value)
                 {
                     trace!("assertion on {:?} should be {:?}", value, expected);
-                    *cond = self.operand_from_scalar(
-                        value_const,
-                        self.tcx.types.bool,
-                        source_info.span,
-                    );
+                    *cond = self.operand_from_scalar(value_const, self.tcx.types.bool);
                 }
             }
             TerminatorKind::SwitchInt { ref mut discr, .. } => {
@@ -1003,11 +949,10 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
             // None of these have Operands to const-propagate.
             TerminatorKind::Goto { .. }
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::Drop { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::Yield { .. }
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::FalseEdge { .. }
@@ -1027,22 +972,30 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
         // We remove all Locals which are restricted in propagation to their containing blocks and
         // which were modified in the current block.
         // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`.
-        let mut locals = std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals);
-        for &local in locals.iter() {
+        let mut written_only_inside_own_block_locals =
+            std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals);
+
+        // This loop can get very hot for some bodies: it check each local in each bb.
+        // To avoid this quadratic behaviour, we only clear the locals that were modified inside
+        // the current block.
+        for local in written_only_inside_own_block_locals.drain() {
+            debug_assert_eq!(
+                self.ecx.machine.can_const_prop[local],
+                ConstPropMode::OnlyInsideOwnBlock
+            );
             Self::remove_const(&mut self.ecx, local);
         }
-        locals.clear();
-        // Put it back so we reuse the heap of the storage
-        self.ecx.machine.written_only_inside_own_block_locals = locals;
+        self.ecx.machine.written_only_inside_own_block_locals =
+            written_only_inside_own_block_locals;
+
         if cfg!(debug_assertions) {
-            // Ensure we are correctly erasing locals with the non-debug-assert logic.
-            for local in self.ecx.machine.only_propagate_inside_block_locals.iter() {
-                assert!(
-                    self.get_const(local.into()).is_none()
-                        || self
-                            .layout_of(self.local_decls[local].ty)
-                            .map_or(true, |layout| layout.is_zst())
-                )
+            for (local, &mode) in self.ecx.machine.can_const_prop.iter_enumerated() {
+                match mode {
+                    ConstPropMode::FullConstProp => {}
+                    ConstPropMode::NoPropagation | ConstPropMode::OnlyInsideOwnBlock => {
+                        self.ensure_not_propagated(local);
+                    }
+                }
             }
         }
     }
diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs
index fd9475748..699fe4489 100644
--- a/compiler/rustc_mir_transform/src/const_prop_lint.rs
+++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs
@@ -1,24 +1,17 @@
 //! Propagates constants for early reporting of statically known
 //! assertion failures
 
-use std::cell::Cell;
-
-use either::{Left, Right};
+use either::Left;
 
 use rustc_const_eval::interpret::Immediate;
 use rustc_const_eval::interpret::{
-    self, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup,
+    self, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup,
 };
 use rustc_hir::def::DefKind;
 use rustc_hir::HirId;
 use rustc_index::bit_set::BitSet;
-use rustc_index::vec::IndexVec;
 use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{
-    AssertKind, BinOp, Body, Constant, Local, LocalDecl, Location, Operand, Place, Rvalue,
-    SourceInfo, SourceScope, SourceScopeData, Statement, StatementKind, Terminator, TerminatorKind,
-    UnOp, RETURN_PLACE,
-};
+use rustc_middle::mir::*;
 use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout};
 use rustc_middle::ty::InternalSubsts;
 use rustc_middle::ty::{
@@ -98,10 +91,7 @@ impl<'tcx> MirLint<'tcx> for ConstProp {
             .predicates
             .iter()
             .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None });
-        if traits::impossible_predicates(
-            tcx,
-            traits::elaborate_predicates(tcx, predicates).map(|o| o.predicate).collect(),
-        ) {
+        if traits::impossible_predicates(tcx, traits::elaborate(tcx, predicates).collect()) {
             trace!("ConstProp skipped for {:?}: found unsatisfiable predicates", def_id);
             return;
         }
@@ -110,7 +100,7 @@ impl<'tcx> MirLint<'tcx> for ConstProp {
 
         let dummy_body = &Body::new(
             body.source,
-            (*body.basic_blocks).clone(),
+            (*body.basic_blocks).to_owned(),
             body.source_scopes.clone(),
             body.local_decls.clone(),
             Default::default(),
@@ -137,11 +127,8 @@ struct ConstPropagator<'mir, 'tcx> {
     ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>,
     tcx: TyCtxt<'tcx>,
     param_env: ParamEnv<'tcx>,
-    source_scopes: &'mir IndexVec<SourceScope, SourceScopeData<'tcx>>,
-    local_decls: &'mir IndexVec<Local, LocalDecl<'tcx>>,
-    // Because we have `MutVisitor` we can't obtain the `SourceInfo` from a `Location`. So we store
-    // the last known `SourceInfo` here and just keep revisiting it.
-    source_info: Option<SourceInfo>,
+    worklist: Vec<BasicBlock>,
+    visited_blocks: BitSet<BasicBlock>,
 }
 
 impl<'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'_, 'tcx> {
@@ -185,17 +172,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         let param_env = tcx.param_env_reveal_all_normalized(def_id);
 
         let can_const_prop = CanConstProp::check(tcx, param_env, body);
-        let mut only_propagate_inside_block_locals = BitSet::new_empty(can_const_prop.len());
-        for (l, mode) in can_const_prop.iter_enumerated() {
-            if *mode == ConstPropMode::OnlyInsideOwnBlock {
-                only_propagate_inside_block_locals.insert(l);
-            }
-        }
         let mut ecx = InterpCx::new(
             tcx,
             tcx.def_span(def_id),
             param_env,
-            ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop),
+            ConstPropMachine::new(can_const_prop),
         );
 
         let ret_layout = ecx
@@ -226,12 +207,19 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             ecx,
             tcx,
             param_env,
-            source_scopes: &dummy_body.source_scopes,
-            local_decls: &dummy_body.local_decls,
-            source_info: None,
+            worklist: vec![START_BLOCK],
+            visited_blocks: BitSet::new_empty(body.basic_blocks.len()),
         }
     }
 
+    fn body(&self) -> &'mir Body<'tcx> {
+        self.ecx.frame().body
+    }
+
+    fn local_decls(&self) -> &'mir LocalDecls<'tcx> {
+        &self.body().local_decls
+    }
+
     fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
         let op = match self.ecx.eval_place_to_op(place, None) {
             Ok(op) => {
@@ -258,22 +246,21 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
     /// Remove `local` from the pool of `Locals`. Allows writing to them,
     /// but not reading from them anymore.
     fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) {
-        ecx.frame_mut().locals[local] = LocalState {
-            value: LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)),
-            layout: Cell::new(None),
-        };
+        ecx.frame_mut().locals[local].value =
+            LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit));
+        ecx.machine.written_only_inside_own_block_locals.remove(&local);
     }
 
     fn lint_root(&self, source_info: SourceInfo) -> Option<HirId> {
-        source_info.scope.lint_root(self.source_scopes)
+        source_info.scope.lint_root(&self.body().source_scopes)
     }
 
-    fn use_ecx<F, T>(&mut self, source_info: SourceInfo, f: F) -> Option<T>
+    fn use_ecx<F, T>(&mut self, location: Location, f: F) -> Option<T>
     where
         F: FnOnce(&mut Self) -> InterpResult<'tcx, T>,
     {
         // Overwrite the PC -- whatever the interpreter does to it does not make any sense anyway.
-        self.ecx.frame_mut().loc = Right(source_info.span);
+        self.ecx.frame_mut().loc = Left(location);
         match f(self) {
             Ok(val) => Some(val),
             Err(error) => {
@@ -292,7 +279,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
     }
 
     /// Returns the value, if any, of evaluating `c`.
-    fn eval_constant(&mut self, c: &Constant<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> {
+    fn eval_constant(&mut self, c: &Constant<'tcx>, location: Location) -> Option<OpTy<'tcx>> {
         // FIXME we need to revisit this for #67176
         if c.needs_subst() {
             return None;
@@ -306,45 +293,41 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         // manually normalized.
         let val = self.tcx.try_normalize_erasing_regions(self.param_env, c.literal).ok()?;
 
-        self.use_ecx(source_info, |this| this.ecx.eval_mir_constant(&val, Some(c.span), None))
+        self.use_ecx(location, |this| this.ecx.eval_mir_constant(&val, Some(c.span), None))
     }
 
     /// Returns the value, if any, of evaluating `place`.
-    fn eval_place(&mut self, place: Place<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> {
+    fn eval_place(&mut self, place: Place<'tcx>, location: Location) -> Option<OpTy<'tcx>> {
         trace!("eval_place(place={:?})", place);
-        self.use_ecx(source_info, |this| this.ecx.eval_place_to_op(place, None))
+        self.use_ecx(location, |this| this.ecx.eval_place_to_op(place, None))
     }
 
     /// Returns the value, if any, of evaluating `op`. Calls upon `eval_constant`
     /// or `eval_place`, depending on the variant of `Operand` used.
-    fn eval_operand(&mut self, op: &Operand<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> {
+    fn eval_operand(&mut self, op: &Operand<'tcx>, location: Location) -> Option<OpTy<'tcx>> {
         match *op {
-            Operand::Constant(ref c) => self.eval_constant(c, source_info),
-            Operand::Move(place) | Operand::Copy(place) => self.eval_place(place, source_info),
+            Operand::Constant(ref c) => self.eval_constant(c, location),
+            Operand::Move(place) | Operand::Copy(place) => self.eval_place(place, location),
         }
     }
 
     fn report_assert_as_lint(
         &self,
         lint: &'static lint::Lint,
-        source_info: SourceInfo,
+        location: Location,
         message: &'static str,
         panic: AssertKind<impl std::fmt::Debug>,
     ) {
-        if let Some(lint_root) = self.lint_root(source_info) {
+        let source_info = self.body().source_info(location);
+        if let Some(lint_root) = self.lint_root(*source_info) {
             self.tcx.struct_span_lint_hir(lint, lint_root, source_info.span, message, |lint| {
                 lint.span_label(source_info.span, format!("{:?}", panic))
             });
         }
     }
 
-    fn check_unary_op(
-        &mut self,
-        op: UnOp,
-        arg: &Operand<'tcx>,
-        source_info: SourceInfo,
-    ) -> Option<()> {
-        if let (val, true) = self.use_ecx(source_info, |this| {
+    fn check_unary_op(&mut self, op: UnOp, arg: &Operand<'tcx>, location: Location) -> Option<()> {
+        if let (val, true) = self.use_ecx(location, |this| {
             let val = this.ecx.read_immediate(&this.ecx.eval_operand(arg, None)?)?;
             let (_res, overflow, _ty) = this.ecx.overflowing_unary_op(op, &val)?;
             Ok((val, overflow))
@@ -354,7 +337,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             assert_eq!(op, UnOp::Neg, "Neg is the only UnOp that can overflow");
             self.report_assert_as_lint(
                 lint::builtin::ARITHMETIC_OVERFLOW,
-                source_info,
+                location,
                 "this arithmetic operation will overflow",
                 AssertKind::OverflowNeg(val.to_const_int()),
             );
@@ -369,28 +352,27 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         op: BinOp,
         left: &Operand<'tcx>,
         right: &Operand<'tcx>,
-        source_info: SourceInfo,
+        location: Location,
     ) -> Option<()> {
-        let r = self.use_ecx(source_info, |this| {
+        let r = self.use_ecx(location, |this| {
             this.ecx.read_immediate(&this.ecx.eval_operand(right, None)?)
         });
-        let l = self.use_ecx(source_info, |this| {
-            this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?)
-        });
+        let l = self
+            .use_ecx(location, |this| this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?));
         // Check for exceeding shifts *even if* we cannot evaluate the LHS.
         if matches!(op, BinOp::Shr | BinOp::Shl) {
             let r = r.clone()?;
             // We need the type of the LHS. We cannot use `place_layout` as that is the type
             // of the result, which for checked binops is not the same!
-            let left_ty = left.ty(self.local_decls, self.tcx);
+            let left_ty = left.ty(self.local_decls(), self.tcx);
             let left_size = self.ecx.layout_of(left_ty).ok()?.size;
             let right_size = r.layout.size;
             let r_bits = r.to_scalar().to_bits(right_size).ok();
             if r_bits.map_or(false, |b| b >= left_size.bits() as u128) {
-                debug!("check_binary_op: reporting assert for {:?}", source_info);
+                debug!("check_binary_op: reporting assert for {:?}", location);
                 self.report_assert_as_lint(
                     lint::builtin::ARITHMETIC_OVERFLOW,
-                    source_info,
+                    location,
                     "this arithmetic operation will overflow",
                     AssertKind::Overflow(
                         op,
@@ -412,13 +394,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
 
         if let (Some(l), Some(r)) = (l, r) {
             // The remaining operators are handled through `overflowing_binary_op`.
-            if self.use_ecx(source_info, |this| {
+            if self.use_ecx(location, |this| {
                 let (_res, overflow, _ty) = this.ecx.overflowing_binary_op(op, &l, &r)?;
                 Ok(overflow)
             })? {
                 self.report_assert_as_lint(
                     lint::builtin::ARITHMETIC_OVERFLOW,
-                    source_info,
+                    location,
                     "this arithmetic operation will overflow",
                     AssertKind::Overflow(op, l.to_const_int(), r.to_const_int()),
                 );
@@ -428,12 +410,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         Some(())
     }
 
-    fn const_prop(
-        &mut self,
-        rvalue: &Rvalue<'tcx>,
-        source_info: SourceInfo,
-        place: Place<'tcx>,
-    ) -> Option<()> {
+    fn check_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) -> Option<()> {
         // Perform any special handling for specific Rvalue types.
         // Generally, checks here fall into one of two categories:
         //   1. Additional checking to provide useful lints to the user
@@ -448,11 +425,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             // lint.
             Rvalue::UnaryOp(op, arg) => {
                 trace!("checking UnaryOp(op = {:?}, arg = {:?})", op, arg);
-                self.check_unary_op(*op, arg, source_info)?;
+                self.check_unary_op(*op, arg, location)?;
             }
             Rvalue::BinaryOp(op, box (left, right)) => {
                 trace!("checking BinaryOp(op = {:?}, left = {:?}, right = {:?})", op, left, right);
-                self.check_binary_op(*op, left, right, source_info)?;
+                self.check_binary_op(*op, left, right, location)?;
             }
             Rvalue::CheckedBinaryOp(op, box (left, right)) => {
                 trace!(
@@ -461,7 +438,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                     left,
                     right
                 );
-                self.check_binary_op(*op, left, right, source_info)?;
+                self.check_binary_op(*op, left, right, location)?;
             }
 
             // Do not try creating references (#67862)
@@ -500,22 +477,105 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         if rvalue.needs_subst() {
             return None;
         }
-        if !rvalue
-            .ty(&self.ecx.frame().body.local_decls, *self.ecx.tcx)
-            .is_sized(*self.ecx.tcx, self.param_env)
-        {
+        if !rvalue.ty(self.local_decls(), self.tcx).is_sized(self.tcx, self.param_env) {
             // the interpreter doesn't support unsized locals (only unsized arguments),
             // but rustc does (in a kinda broken way), so we have to skip them here
             return None;
         }
 
-        self.use_ecx(source_info, |this| this.ecx.eval_rvalue_into_place(rvalue, place))
+        Some(())
+    }
+
+    fn check_assertion(
+        &mut self,
+        expected: bool,
+        msg: &AssertKind<Operand<'tcx>>,
+        cond: &Operand<'tcx>,
+        location: Location,
+    ) -> Option<!> {
+        let ref value = self.eval_operand(&cond, location)?;
+        trace!("assertion on {:?} should be {:?}", value, expected);
+
+        let expected = Scalar::from_bool(expected);
+        let value_const = self.use_ecx(location, |this| this.ecx.read_scalar(&value))?;
+
+        if expected != value_const {
+            // Poison all places this operand references so that further code
+            // doesn't use the invalid value
+            if let Some(place) = cond.place() {
+                Self::remove_const(&mut self.ecx, place.local);
+            }
+
+            enum DbgVal<T> {
+                Val(T),
+                Underscore,
+            }
+            impl<T: std::fmt::Debug> std::fmt::Debug for DbgVal<T> {
+                fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                    match self {
+                        Self::Val(val) => val.fmt(fmt),
+                        Self::Underscore => fmt.write_str("_"),
+                    }
+                }
+            }
+            let mut eval_to_int = |op| {
+                // This can be `None` if the lhs wasn't const propagated and we just
+                // triggered the assert on the value of the rhs.
+                self.eval_operand(op, location)
+                    .and_then(|op| self.ecx.read_immediate(&op).ok())
+                    .map_or(DbgVal::Underscore, |op| DbgVal::Val(op.to_const_int()))
+            };
+            let msg = match msg {
+                AssertKind::DivisionByZero(op) => AssertKind::DivisionByZero(eval_to_int(op)),
+                AssertKind::RemainderByZero(op) => AssertKind::RemainderByZero(eval_to_int(op)),
+                AssertKind::Overflow(bin_op @ (BinOp::Div | BinOp::Rem), op1, op2) => {
+                    // Division overflow is *UB* in the MIR, and different than the
+                    // other overflow checks.
+                    AssertKind::Overflow(*bin_op, eval_to_int(op1), eval_to_int(op2))
+                }
+                AssertKind::BoundsCheck { ref len, ref index } => {
+                    let len = eval_to_int(len);
+                    let index = eval_to_int(index);
+                    AssertKind::BoundsCheck { len, index }
+                }
+                // Remaining overflow errors are already covered by checks on the binary operators.
+                AssertKind::Overflow(..) | AssertKind::OverflowNeg(_) => return None,
+                // Need proper const propagator for these.
+                _ => return None,
+            };
+            self.report_assert_as_lint(
+                lint::builtin::UNCONDITIONAL_PANIC,
+                location,
+                "this operation will panic at runtime",
+                msg,
+            );
+        }
+
+        None
+    }
+
+    fn ensure_not_propagated(&self, local: Local) {
+        if cfg!(debug_assertions) {
+            assert!(
+                self.get_const(local.into()).is_none()
+                    || self
+                        .layout_of(self.local_decls()[local].ty)
+                        .map_or(true, |layout| layout.is_zst()),
+                "failed to remove values for `{local:?}`, value={:?}",
+                self.get_const(local.into()),
+            )
+        }
     }
 }
 
 impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> {
     fn visit_body(&mut self, body: &Body<'tcx>) {
-        for (bb, data) in body.basic_blocks.iter_enumerated() {
+        while let Some(bb) = self.worklist.pop() {
+            if !self.visited_blocks.insert(bb) {
+                continue;
+            }
+
+            let data = &body.basic_blocks[bb];
             self.visit_basic_block_data(bb, data);
         }
     }
@@ -527,202 +587,147 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> {
     fn visit_constant(&mut self, constant: &Constant<'tcx>, location: Location) {
         trace!("visit_constant: {:?}", constant);
         self.super_constant(constant, location);
-        self.eval_constant(constant, self.source_info.unwrap());
+        self.eval_constant(constant, location);
+    }
+
+    fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) {
+        self.super_assign(place, rvalue, location);
+
+        let Some(()) = self.check_rvalue(rvalue, location) else { return };
+
+        match self.ecx.machine.can_const_prop[place.local] {
+            // Do nothing if the place is indirect.
+            _ if place.is_indirect() => {}
+            ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local),
+            ConstPropMode::OnlyInsideOwnBlock | ConstPropMode::FullConstProp => {
+                if self
+                    .use_ecx(location, |this| this.ecx.eval_rvalue_into_place(rvalue, *place))
+                    .is_none()
+                {
+                    // Const prop failed, so erase the destination, ensuring that whatever happens
+                    // from here on, does not know about the previous value.
+                    // This is important in case we have
+                    // ```rust
+                    // let mut x = 42;
+                    // x = SOME_MUTABLE_STATIC;
+                    // // x must now be uninit
+                    // ```
+                    // FIXME: we overzealously erase the entire local, because that's easier to
+                    // implement.
+                    trace!(
+                        "propagation into {:?} failed.
+                        Nuking the entire site from orbit, it's the only way to be sure",
+                        place,
+                    );
+                    Self::remove_const(&mut self.ecx, place.local);
+                }
+            }
+        }
     }
 
     fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
         trace!("visit_statement: {:?}", statement);
-        let source_info = statement.source_info;
-        self.source_info = Some(source_info);
-        if let StatementKind::Assign(box (place, ref rval)) = statement.kind {
-            let can_const_prop = self.ecx.machine.can_const_prop[place.local];
-            if let Some(()) = self.const_prop(rval, source_info, place) {
-                match can_const_prop {
-                    ConstPropMode::OnlyInsideOwnBlock => {
-                        trace!(
-                            "found local restricted to its block. \
-                                Will remove it from const-prop after block is finished. Local: {:?}",
-                            place.local
-                        );
-                    }
-                    ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => {
-                        trace!("can't propagate into {:?}", place);
-                        if place.local != RETURN_PLACE {
+
+        // We want to evaluate operands before any change to the assigned-to value,
+        // so we recurse first.
+        self.super_statement(statement, location);
+
+        match statement.kind {
+            StatementKind::SetDiscriminant { ref place, .. } => {
+                match self.ecx.machine.can_const_prop[place.local] {
+                    // Do nothing if the place is indirect.
+                    _ if place.is_indirect() => {}
+                    ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local),
+                    ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => {
+                        if self.use_ecx(location, |this| this.ecx.statement(statement)).is_some() {
+                            trace!("propped discriminant into {:?}", place);
+                        } else {
                             Self::remove_const(&mut self.ecx, place.local);
                         }
                     }
-                    ConstPropMode::FullConstProp => {}
                 }
-            } else {
-                // Const prop failed, so erase the destination, ensuring that whatever happens
-                // from here on, does not know about the previous value.
-                // This is important in case we have
-                // ```rust
-                // let mut x = 42;
-                // x = SOME_MUTABLE_STATIC;
-                // // x must now be uninit
-                // ```
-                // FIXME: we overzealously erase the entire local, because that's easier to
-                // implement.
-                trace!(
-                    "propagation into {:?} failed.
-                        Nuking the entire site from orbit, it's the only way to be sure",
-                    place,
-                );
-                Self::remove_const(&mut self.ecx, place.local);
             }
-        } else {
-            match statement.kind {
-                StatementKind::SetDiscriminant { ref place, .. } => {
-                    match self.ecx.machine.can_const_prop[place.local] {
-                        ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => {
-                            if self
-                                .use_ecx(source_info, |this| this.ecx.statement(statement))
-                                .is_some()
-                            {
-                                trace!("propped discriminant into {:?}", place);
-                            } else {
-                                Self::remove_const(&mut self.ecx, place.local);
-                            }
-                        }
-                        ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => {
-                            Self::remove_const(&mut self.ecx, place.local);
-                        }
-                    }
-                }
-                StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => {
-                    let frame = self.ecx.frame_mut();
-                    frame.locals[local].value =
-                        if let StatementKind::StorageLive(_) = statement.kind {
-                            LocalValue::Live(interpret::Operand::Immediate(
-                                interpret::Immediate::Uninit,
-                            ))
-                        } else {
-                            LocalValue::Dead
-                        };
-                }
-                _ => {}
+            StatementKind::StorageLive(local) => {
+                let frame = self.ecx.frame_mut();
+                frame.locals[local].value =
+                    LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit));
+            }
+            StatementKind::StorageDead(local) => {
+                let frame = self.ecx.frame_mut();
+                frame.locals[local].value = LocalValue::Dead;
             }
+            _ => {}
         }
-
-        self.super_statement(statement, location);
     }
 
     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
-        let source_info = terminator.source_info;
-        self.source_info = Some(source_info);
         self.super_terminator(terminator, location);
         match &terminator.kind {
             TerminatorKind::Assert { expected, ref msg, ref cond, .. } => {
-                if let Some(ref value) = self.eval_operand(&cond, source_info) {
-                    trace!("assertion on {:?} should be {:?}", value, expected);
-                    let expected = Scalar::from_bool(*expected);
-                    let Ok(value_const) = self.ecx.read_scalar(&value) else {
-                        // FIXME should be used use_ecx rather than a local match... but we have
-                        // quite a few of these read_scalar/read_immediate that need fixing.
-                        return
-                    };
-                    if expected != value_const {
-                        enum DbgVal<T> {
-                            Val(T),
-                            Underscore,
-                        }
-                        impl<T: std::fmt::Debug> std::fmt::Debug for DbgVal<T> {
-                            fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                                match self {
-                                    Self::Val(val) => val.fmt(fmt),
-                                    Self::Underscore => fmt.write_str("_"),
-                                }
-                            }
-                        }
-                        let mut eval_to_int = |op| {
-                            // This can be `None` if the lhs wasn't const propagated and we just
-                            // triggered the assert on the value of the rhs.
-                            self.eval_operand(op, source_info)
-                                .and_then(|op| self.ecx.read_immediate(&op).ok())
-                                .map_or(DbgVal::Underscore, |op| DbgVal::Val(op.to_const_int()))
-                        };
-                        let msg = match msg {
-                            AssertKind::DivisionByZero(op) => {
-                                Some(AssertKind::DivisionByZero(eval_to_int(op)))
-                            }
-                            AssertKind::RemainderByZero(op) => {
-                                Some(AssertKind::RemainderByZero(eval_to_int(op)))
-                            }
-                            AssertKind::Overflow(bin_op @ (BinOp::Div | BinOp::Rem), op1, op2) => {
-                                // Division overflow is *UB* in the MIR, and different than the
-                                // other overflow checks.
-                                Some(AssertKind::Overflow(
-                                    *bin_op,
-                                    eval_to_int(op1),
-                                    eval_to_int(op2),
-                                ))
-                            }
-                            AssertKind::BoundsCheck { ref len, ref index } => {
-                                let len = eval_to_int(len);
-                                let index = eval_to_int(index);
-                                Some(AssertKind::BoundsCheck { len, index })
-                            }
-                            // Remaining overflow errors are already covered by checks on the binary operators.
-                            AssertKind::Overflow(..) | AssertKind::OverflowNeg(_) => None,
-                            // Need proper const propagator for these.
-                            _ => None,
-                        };
-                        // Poison all places this operand references so that further code
-                        // doesn't use the invalid value
-                        match cond {
-                            Operand::Move(ref place) | Operand::Copy(ref place) => {
-                                Self::remove_const(&mut self.ecx, place.local);
-                            }
-                            Operand::Constant(_) => {}
-                        }
-                        if let Some(msg) = msg {
-                            self.report_assert_as_lint(
-                                lint::builtin::UNCONDITIONAL_PANIC,
-                                source_info,
-                                "this operation will panic at runtime",
-                                msg,
-                            );
-                        }
-                    }
+                self.check_assertion(*expected, msg, cond, location);
+            }
+            TerminatorKind::SwitchInt { ref discr, ref targets } => {
+                if let Some(ref value) = self.eval_operand(&discr, location)
+                  && let Some(value_const) = self.use_ecx(location, |this| this.ecx.read_scalar(&value))
+                  && let Ok(constant) = value_const.try_to_int()
+                  && let Ok(constant) = constant.to_bits(constant.size())
+                {
+                    // We managed to evaluate the discriminant, so we know we only need to visit
+                    // one target.
+                    let target = targets.target_for_value(constant);
+                    self.worklist.push(target);
+                    return;
                 }
+                // We failed to evaluate the discriminant, fallback to visiting all successors.
             }
             // None of these have Operands to const-propagate.
             TerminatorKind::Goto { .. }
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::Drop { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::Yield { .. }
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::FalseEdge { .. }
             | TerminatorKind::FalseUnwind { .. }
-            | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::Call { .. }
             | TerminatorKind::InlineAsm { .. } => {}
         }
 
+        self.worklist.extend(terminator.successors());
+    }
+
+    fn visit_basic_block_data(&mut self, block: BasicBlock, data: &BasicBlockData<'tcx>) {
+        self.super_basic_block_data(block, data);
+
         // We remove all Locals which are restricted in propagation to their containing blocks and
         // which were modified in the current block.
         // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`.
-        let mut locals = std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals);
-        for &local in locals.iter() {
+        let mut written_only_inside_own_block_locals =
+            std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals);
+
+        // This loop can get very hot for some bodies: it check each local in each bb.
+        // To avoid this quadratic behaviour, we only clear the locals that were modified inside
+        // the current block.
+        for local in written_only_inside_own_block_locals.drain() {
+            debug_assert_eq!(
+                self.ecx.machine.can_const_prop[local],
+                ConstPropMode::OnlyInsideOwnBlock
+            );
             Self::remove_const(&mut self.ecx, local);
         }
-        locals.clear();
-        // Put it back so we reuse the heap of the storage
-        self.ecx.machine.written_only_inside_own_block_locals = locals;
+        self.ecx.machine.written_only_inside_own_block_locals =
+            written_only_inside_own_block_locals;
+
         if cfg!(debug_assertions) {
-            // Ensure we are correctly erasing locals with the non-debug-assert logic.
-            for local in self.ecx.machine.only_propagate_inside_block_locals.iter() {
-                assert!(
-                    self.get_const(local.into()).is_none()
-                        || self
-                            .layout_of(self.local_decls[local].ty)
-                            .map_or(true, |layout| layout.is_zst())
-                )
+            for (local, &mode) in self.ecx.machine.can_const_prop.iter_enumerated() {
+                match mode {
+                    ConstPropMode::FullConstProp => {}
+                    ConstPropMode::NoPropagation | ConstPropMode::OnlyInsideOwnBlock => {
+                        self.ensure_not_propagated(local);
+                    }
+                }
             }
         }
     }
diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs
index f27beb64a..b571215f2 100644
--- a/compiler/rustc_mir_transform/src/copy_prop.rs
+++ b/compiler/rustc_mir_transform/src/copy_prop.rs
@@ -1,5 +1,5 @@
 use rustc_index::bit_set::BitSet;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::IndexSlice;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
@@ -102,7 +102,7 @@ struct Replacer<'a, 'tcx> {
     fully_moved: BitSet<Local>,
     storage_to_remove: BitSet<Local>,
     borrowed_locals: BitSet<Local>,
-    copy_classes: &'a IndexVec<Local, Local>,
+    copy_classes: &'a IndexSlice<Local, Local>,
 }
 
 impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> {
@@ -131,7 +131,6 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> {
             PlaceContext::NonMutatingUse(
                 NonMutatingUseContext::SharedBorrow
                 | NonMutatingUseContext::ShallowBorrow
-                | NonMutatingUseContext::UniqueBorrow
                 | NonMutatingUseContext::AddressOf,
             ) => true,
             // For debuginfo, merging locals is ok.
diff --git a/compiler/rustc_mir_transform/src/coverage/debug.rs b/compiler/rustc_mir_transform/src/coverage/debug.rs
index 22ea8710e..725883b83 100644
--- a/compiler/rustc_mir_transform/src/coverage/debug.rs
+++ b/compiler/rustc_mir_transform/src/coverage/debug.rs
@@ -818,11 +818,10 @@ pub(super) fn term_type(kind: &TerminatorKind<'_>) -> &'static str {
         TerminatorKind::Goto { .. } => "Goto",
         TerminatorKind::SwitchInt { .. } => "SwitchInt",
         TerminatorKind::Resume => "Resume",
-        TerminatorKind::Abort => "Abort",
+        TerminatorKind::Terminate => "Terminate",
         TerminatorKind::Return => "Return",
         TerminatorKind::Unreachable => "Unreachable",
         TerminatorKind::Drop { .. } => "Drop",
-        TerminatorKind::DropAndReplace { .. } => "DropAndReplace",
         TerminatorKind::Call { .. } => "Call",
         TerminatorKind::Assert { .. } => "Assert",
         TerminatorKind::Yield { .. } => "Yield",
diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs
index a2671eef2..7391a77b0 100644
--- a/compiler/rustc_mir_transform/src/coverage/graph.rs
+++ b/compiler/rustc_mir_transform/src/coverage/graph.rs
@@ -5,7 +5,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::graph::dominators::{self, Dominators};
 use rustc_data_structures::graph::{self, GraphSuccessors, WithNumNodes, WithStartNode};
 use rustc_index::bit_set::BitSet;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::mir::coverage::*;
 use rustc_middle::mir::{self, BasicBlock, BasicBlockData, Terminator, TerminatorKind};
 
@@ -37,8 +37,7 @@ impl CoverageGraph {
         // `SwitchInt` to have multiple targets to the same destination `BasicBlock`, so
         // de-duplication is required. This is done without reordering the successors.
 
-        let bcbs_len = bcbs.len();
-        let mut seen = IndexVec::from_elem_n(false, bcbs_len);
+        let mut seen = IndexVec::from_elem(false, &bcbs);
         let successors = IndexVec::from_fn_n(
             |bcb| {
                 for b in seen.iter_mut() {
@@ -60,7 +59,7 @@ impl CoverageGraph {
             bcbs.len(),
         );
 
-        let mut predecessors = IndexVec::from_elem_n(Vec::new(), bcbs.len());
+        let mut predecessors = IndexVec::from_elem(Vec::new(), &bcbs);
         for (bcb, bcb_successors) in successors.iter_enumerated() {
             for &successor in bcb_successors {
                 predecessors[successor].push(bcb);
@@ -123,7 +122,7 @@ impl CoverageGraph {
 
             match term.kind {
                 TerminatorKind::Return { .. }
-                | TerminatorKind::Abort
+                | TerminatorKind::Terminate
                 | TerminatorKind::Yield { .. }
                 | TerminatorKind::SwitchInt { .. } => {
                     // The `bb` has more than one _outgoing_ edge, or exits the function. Save the
@@ -137,7 +136,7 @@ impl CoverageGraph {
                     debug!("  because term.kind = {:?}", term.kind);
                     // Note that this condition is based on `TerminatorKind`, even though it
                     // theoretically boils down to `successors().len() != 1`; that is, either zero
-                    // (e.g., `Return`, `Abort`) or multiple successors (e.g., `SwitchInt`), but
+                    // (e.g., `Return`, `Terminate`) or multiple successors (e.g., `SwitchInt`), but
                     // since the BCB CFG ignores things like unwind branches (which exist in the
                     // `Terminator`s `successors()` list) checking the number of successors won't
                     // work.
@@ -156,7 +155,6 @@ impl CoverageGraph {
                 | TerminatorKind::Resume
                 | TerminatorKind::Unreachable
                 | TerminatorKind::Drop { .. }
-                | TerminatorKind::DropAndReplace { .. }
                 | TerminatorKind::Call { .. }
                 | TerminatorKind::GeneratorDrop
                 | TerminatorKind::Assert { .. }
@@ -177,10 +175,10 @@ impl CoverageGraph {
 
     fn add_basic_coverage_block(
         bcbs: &mut IndexVec<BasicCoverageBlock, BasicCoverageBlockData>,
-        bb_to_bcb: &mut IndexVec<BasicBlock, Option<BasicCoverageBlock>>,
+        bb_to_bcb: &mut IndexSlice<BasicBlock, Option<BasicCoverageBlock>>,
         basic_blocks: Vec<BasicBlock>,
     ) {
-        let bcb = BasicCoverageBlock::from_usize(bcbs.len());
+        let bcb = bcbs.next_index();
         for &bb in basic_blocks.iter() {
             bb_to_bcb[bb] = Some(bcb);
         }
@@ -538,29 +536,29 @@ impl TraverseCoverageGraphWithLoops {
             "TraverseCoverageGraphWithLoops::next - context_stack: {:?}",
             self.context_stack.iter().rev().collect::<Vec<_>>()
         );
-        while let Some(next_bcb) = {
-            // Strip contexts with empty worklists from the top of the stack
-            while self.context_stack.last().map_or(false, |context| context.worklist.is_empty()) {
+
+        while let Some(context) = self.context_stack.last_mut() {
+            if let Some(next_bcb) = context.worklist.pop() {
+                if !self.visited.insert(next_bcb) {
+                    debug!("Already visited: {:?}", next_bcb);
+                    continue;
+                }
+                debug!("Visiting {:?}", next_bcb);
+                if self.backedges[next_bcb].len() > 0 {
+                    debug!("{:?} is a loop header! Start a new TraversalContext...", next_bcb);
+                    self.context_stack.push(TraversalContext {
+                        loop_backedges: Some((self.backedges[next_bcb].clone(), next_bcb)),
+                        worklist: Vec::new(),
+                    });
+                }
+                self.extend_worklist(basic_coverage_blocks, next_bcb);
+                return Some(next_bcb);
+            } else {
+                // Strip contexts with empty worklists from the top of the stack
                 self.context_stack.pop();
             }
-            // Pop the next bcb off of the current context_stack. If none, all BCBs were visited.
-            self.context_stack.last_mut().map_or(None, |context| context.worklist.pop())
-        } {
-            if !self.visited.insert(next_bcb) {
-                debug!("Already visited: {:?}", next_bcb);
-                continue;
-            }
-            debug!("Visiting {:?}", next_bcb);
-            if self.backedges[next_bcb].len() > 0 {
-                debug!("{:?} is a loop header! Start a new TraversalContext...", next_bcb);
-                self.context_stack.push(TraversalContext {
-                    loop_backedges: Some((self.backedges[next_bcb].clone(), next_bcb)),
-                    worklist: Vec::new(),
-                });
-            }
-            self.extend_worklist(basic_coverage_blocks, next_bcb);
-            return Some(next_bcb);
         }
+
         None
     }
 
diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs
index 9a6171598..5ecb2d6a6 100644
--- a/compiler/rustc_mir_transform/src/coverage/mod.rs
+++ b/compiler/rustc_mir_transform/src/coverage/mod.rs
@@ -577,5 +577,5 @@ fn get_body_span<'tcx>(
 fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx rustc_hir::Body<'tcx>) -> u64 {
     // FIXME(cjgillot) Stop hashing HIR manually here.
     let owner = hir_body.id().hir_id.owner;
-    tcx.hir_owner_nodes(owner).unwrap().hash_including_bodies.to_smaller_hash()
+    tcx.hir_owner_nodes(owner).unwrap().opt_hash_including_bodies.unwrap().to_smaller_hash()
 }
diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index 8ee316773..287ae2170 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -832,6 +832,7 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span>
         | StatementKind::SetDiscriminant { .. }
         | StatementKind::Deinit(..)
         | StatementKind::Retag(_, _)
+        | StatementKind::PlaceMention(..)
         | StatementKind::AscribeUserType(_, _) => {
             Some(statement.source_info.span)
         }
@@ -850,7 +851,6 @@ pub(super) fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Sp
         TerminatorKind::Unreachable // Unreachable blocks are not connected to the MIR CFG
         | TerminatorKind::Assert { .. }
         | TerminatorKind::Drop { .. }
-        | TerminatorKind::DropAndReplace { .. }
         | TerminatorKind::SwitchInt { .. }
         // For `FalseEdge`, only the `real` branch is taken, so it is similar to a `Goto`.
         | TerminatorKind::FalseEdge { .. }
@@ -869,7 +869,7 @@ pub(super) fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Sp
 
         // Retain spans from all other terminators
         TerminatorKind::Resume
-        | TerminatorKind::Abort
+        | TerminatorKind::Terminate
         | TerminatorKind::Return
         | TerminatorKind::Yield { .. }
         | TerminatorKind::GeneratorDrop
diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs
index fa7f22303..0f6c06e37 100644
--- a/compiler/rustc_mir_transform/src/coverage/tests.rs
+++ b/compiler/rustc_mir_transform/src/coverage/tests.rs
@@ -65,7 +65,7 @@ impl<'tcx> MockBlocks<'tcx> {
     }
 
     fn push(&mut self, kind: TerminatorKind<'tcx>) -> BasicBlock {
-        let next_lo = if let Some(last) = self.blocks.last() {
+        let next_lo = if let Some(last) = self.blocks.last_index() {
             self.blocks[last].terminator().source_info.span.hi()
         } else {
             BytePos(1)
@@ -86,7 +86,6 @@ impl<'tcx> MockBlocks<'tcx> {
             TerminatorKind::Assert { ref mut target, .. }
             | TerminatorKind::Call { target: Some(ref mut target), .. }
             | TerminatorKind::Drop { ref mut target, .. }
-            | TerminatorKind::DropAndReplace { ref mut target, .. }
             | TerminatorKind::FalseEdge { real_target: ref mut target, .. }
             | TerminatorKind::FalseUnwind { real_target: ref mut target, .. }
             | TerminatorKind::Goto { ref mut target }
@@ -141,7 +140,7 @@ impl<'tcx> MockBlocks<'tcx> {
                 args: vec![],
                 destination: self.dummy_place.clone(),
                 target: Some(TEMP_BLOCK),
-                cleanup: None,
+                unwind: UnwindAction::Continue,
                 from_hir_call: false,
                 fn_span: DUMMY_SP,
             },
@@ -184,7 +183,6 @@ fn debug_basic_blocks(mir_body: &Body<'_>) -> String {
                     TerminatorKind::Assert { target, .. }
                     | TerminatorKind::Call { target: Some(target), .. }
                     | TerminatorKind::Drop { target, .. }
-                    | TerminatorKind::DropAndReplace { target, .. }
                     | TerminatorKind::FalseEdge { real_target: target, .. }
                     | TerminatorKind::FalseUnwind { real_target: target, .. }
                     | TerminatorKind::Goto { target }
diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
index 49ded10ba..d4db7e2de 100644
--- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
+++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
@@ -8,12 +8,12 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def::DefKind;
 use rustc_middle::mir::visit::{MutVisitor, Visitor};
 use rustc_middle::mir::*;
+use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_mir_dataflow::value_analysis::{Map, State, TrackElem, ValueAnalysis, ValueOrPlace};
 use rustc_mir_dataflow::{lattice::FlatSet, Analysis, ResultsVisitor, SwitchIntEdgeEffects};
 use rustc_span::DUMMY_SP;
-use rustc_target::abi::Align;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{Align, FieldIdx, VariantIdx};
 
 use crate::MirPass;
 
@@ -147,7 +147,7 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> {
                         for (field_index, operand) in operands.iter().enumerate() {
                             if let Some(field) = self.map().apply(
                                 variant_target_idx,
-                                TrackElem::Field(Field::from_usize(field_index)),
+                                TrackElem::Field(FieldIdx::from_usize(field_index)),
                             ) {
                                 let result = self.handle_operand(operand, state);
                                 state.insert_idx(field, result, self.map());
@@ -548,7 +548,7 @@ impl<'mir, 'tcx> rustc_const_eval::interpret::Machine<'mir, 'tcx> for DummyMachi
         unimplemented!()
     }
 
-    fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool {
+    fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>, _layout: TyAndLayout<'tcx>) -> bool {
         unimplemented!()
     }
     fn alignment_check_failed(
@@ -567,7 +567,7 @@ impl<'mir, 'tcx> rustc_const_eval::interpret::Machine<'mir, 'tcx> for DummyMachi
         _args: &[rustc_const_eval::interpret::OpTy<'tcx, Self::Provenance>],
         _destination: &rustc_const_eval::interpret::PlaceTy<'tcx, Self::Provenance>,
         _target: Option<BasicBlock>,
-        _unwind: rustc_const_eval::interpret::StackPopUnwind,
+        _unwind: UnwindAction,
     ) -> interpret::InterpResult<'tcx, Option<(&'mir Body<'tcx>, ty::Instance<'tcx>)>> {
         unimplemented!()
     }
@@ -578,7 +578,7 @@ impl<'mir, 'tcx> rustc_const_eval::interpret::Machine<'mir, 'tcx> for DummyMachi
         _args: &[rustc_const_eval::interpret::OpTy<'tcx, Self::Provenance>],
         _destination: &rustc_const_eval::interpret::PlaceTy<'tcx, Self::Provenance>,
         _target: Option<BasicBlock>,
-        _unwind: rustc_const_eval::interpret::StackPopUnwind,
+        _unwind: UnwindAction,
     ) -> interpret::InterpResult<'tcx> {
         unimplemented!()
     }
@@ -586,7 +586,7 @@ impl<'mir, 'tcx> rustc_const_eval::interpret::Machine<'mir, 'tcx> for DummyMachi
     fn assert_panic(
         _ecx: &mut InterpCx<'mir, 'tcx, Self>,
         _msg: &rustc_middle::mir::AssertMessage<'tcx>,
-        _unwind: Option<BasicBlock>,
+        _unwind: UnwindAction,
     ) -> interpret::InterpResult<'tcx> {
         unimplemented!()
     }
diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
index 9dbfb089d..18c407b42 100644
--- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs
+++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
@@ -56,7 +56,9 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, borrowed: &BitS
                 | StatementKind::ConstEvalCounter
                 | StatementKind::Nop => (),
 
-                StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => {
+                StatementKind::FakeRead(_)
+                | StatementKind::PlaceMention(_)
+                | StatementKind::AscribeUserType(_, _) => {
                     bug!("{:?} not found in this MIR phase!", &statement.kind)
                 }
             }
diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
index 89ca04a15..e5c3fa564 100644
--- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
+++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -5,7 +5,7 @@
 //! purposes on a best-effort basis. We compute them here and store them into the crate metadata so
 //! dependent crates can use them.
 
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::LocalDefId;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::{Body, Local, Location, Operand, Terminator, TerminatorKind, RETURN_PLACE};
@@ -149,7 +149,10 @@ fn type_will_always_be_passed_directly(ty: Ty<'_>) -> bool {
 /// body of the function instead of just the signature. These can be useful for optimization
 /// purposes on a best-effort basis. We compute them here and store them into the crate metadata so
 /// dependent crates can use them.
-pub fn deduced_param_attrs<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx [DeducedParamAttrs] {
+pub fn deduced_param_attrs<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    def_id: LocalDefId,
+) -> &'tcx [DeducedParamAttrs] {
     // This computation is unfortunately rather expensive, so don't do it unless we're optimizing.
     // Also skip it in incremental mode.
     if tcx.sess.opts.optimize == OptLevel::No || tcx.sess.opts.incremental.is_some() {
@@ -182,10 +185,6 @@ pub fn deduced_param_attrs<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx [Ded
         return &[];
     }
 
-    // Deduced attributes for other crates should be read from the metadata instead of via this
-    // function.
-    debug_assert!(def_id.is_local());
-
     // Grab the optimized MIR. Analyze it to determine which arguments have been mutated.
     let body: &Body<'tcx> = tcx.optimized_mir(def_id);
     let mut deduce_read_only = DeduceReadOnly::new(body.arg_count);
diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs
index 7508df92d..b8a5b92be 100644
--- a/compiler/rustc_mir_transform/src/deref_separator.rs
+++ b/compiler/rustc_mir_transform/src/deref_separator.rs
@@ -40,7 +40,7 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> {
                     let temp = self.patcher.new_internal_with_info(
                         ty,
                         self.local_decls[p_ref.local].source_info.span,
-                        Some(Box::new(LocalInfo::DerefTemp)),
+                        LocalInfo::DerefTemp,
                     );
 
                     // We are adding current p_ref's projections to our
diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs
index 2e481b972..391649177 100644
--- a/compiler/rustc_mir_transform/src/dest_prop.rs
+++ b/compiler/rustc_mir_transform/src/dest_prop.rs
@@ -83,7 +83,7 @@
 //!   that ever have their address taken. Of course that requires actually having alias analysis
 //!   (and a model to build it on), so this might be a bit of a ways off.
 //!
-//! * Various perf improvents. There are a bunch of comments in here marked `PERF` with ideas for
+//! * Various perf improvements. There are a bunch of comments in here marked `PERF` with ideas for
 //!   how to do things more efficiently. However, the complexity of the pass as a whole should be
 //!   kept in mind.
 //!
@@ -583,7 +583,9 @@ impl WriteInfo {
             | StatementKind::Coverage(_)
             | StatementKind::StorageLive(_)
             | StatementKind::StorageDead(_) => (),
-            StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => {
+            StatementKind::FakeRead(_)
+            | StatementKind::AscribeUserType(_, _)
+            | StatementKind::PlaceMention(_) => {
                 bug!("{:?} not found in this MIR phase", statement)
             }
         }
@@ -643,15 +645,14 @@ impl WriteInfo {
                 }
             }
             TerminatorKind::Goto { .. }
-            | TerminatorKind::Resume { .. }
-            | TerminatorKind::Abort { .. }
+            | TerminatorKind::Resume
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable { .. } => (),
             TerminatorKind::Drop { .. } => {
                 // `Drop`s create a `&mut` and so are not considered
             }
-            TerminatorKind::DropAndReplace { .. }
-            | TerminatorKind::Yield { .. }
+            TerminatorKind::Yield { .. }
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::FalseEdge { .. }
             | TerminatorKind::FalseUnwind { .. } => {
@@ -787,7 +788,7 @@ impl<'tcx> Visitor<'tcx> for FindAssignments<'_, '_, 'tcx> {
 fn is_local_required(local: Local, body: &Body<'_>) -> bool {
     match body.local_kind(local) {
         LocalKind::Arg | LocalKind::ReturnPointer => true,
-        LocalKind::Var | LocalKind::Temp => false,
+        LocalKind::Temp => false,
     }
 }
 
diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
index 954bb5aff..856234994 100644
--- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
@@ -9,6 +9,7 @@ use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::visit::MutVisitor;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{Ty, TyCtxt};
+use rustc_target::abi::FieldIdx;
 
 /// Constructs the types used when accessing a Box's pointer
 pub fn build_ptr_tys<'tcx>(
@@ -32,9 +33,9 @@ pub fn build_projection<'tcx>(
     ptr_ty: Ty<'tcx>,
 ) -> [PlaceElem<'tcx>; 3] {
     [
-        PlaceElem::Field(Field::new(0), unique_ty),
-        PlaceElem::Field(Field::new(0), nonnull_ty),
-        PlaceElem::Field(Field::new(0), ptr_ty),
+        PlaceElem::Field(FieldIdx::new(0), unique_ty),
+        PlaceElem::Field(FieldIdx::new(0), nonnull_ty),
+        PlaceElem::Field(FieldIdx::new(0), ptr_ty),
     ]
 }
 
@@ -91,13 +92,14 @@ pub struct ElaborateBoxDerefs;
 impl<'tcx> MirPass<'tcx> for ElaborateBoxDerefs {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         if let Some(def_id) = tcx.lang_items().owned_box() {
-            let unique_did = tcx.adt_def(def_id).non_enum_variant().fields[0].did;
+            let unique_did =
+                tcx.adt_def(def_id).non_enum_variant().fields[FieldIdx::from_u32(0)].did;
 
             let Some(nonnull_def) = tcx.type_of(unique_did).subst_identity().ty_adt_def() else {
                 span_bug!(tcx.def_span(unique_did), "expected Box to contain Unique")
             };
 
-            let nonnull_did = nonnull_def.non_enum_variant().fields[0].did;
+            let nonnull_did = nonnull_def.non_enum_variant().fields[FieldIdx::from_u32(0)].did;
 
             let patch = MirPatch::new(body);
 
diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs
index bdfd8dc6e..a702113bd 100644
--- a/compiler/rustc_mir_transform/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs
@@ -14,19 +14,18 @@ use rustc_mir_dataflow::un_derefer::UnDerefer;
 use rustc_mir_dataflow::MoveDataParamEnv;
 use rustc_mir_dataflow::{on_all_children_bits, on_all_drop_children_bits};
 use rustc_mir_dataflow::{Analysis, ResultsCursor};
-use rustc_span::Span;
-use rustc_target::abi::VariantIdx;
+use rustc_span::{DesugaringKind, Span};
+use rustc_target::abi::{FieldIdx, VariantIdx};
 use std::fmt;
 
-/// During MIR building, Drop and DropAndReplace terminators are inserted in every place where a drop may occur.
+/// During MIR building, Drop terminators are inserted in every place where a drop may occur.
 /// However, in this phase, the presence of these terminators does not guarantee that a destructor will run,
 /// as the target of the drop may be uninitialized.
 /// In general, the compiler cannot determine at compile time whether a destructor will run or not.
 ///
-/// At a high level, this pass refines Drop and DropAndReplace to only run the destructor if the
+/// At a high level, this pass refines Drop to only run the destructor if the
 /// target is initialized. The way this is achievied is by inserting drop flags for every variable
 /// that may be dropped, and then using those flags to determine whether a destructor should run.
-/// This pass also removes DropAndReplace, replacing it with a Drop paired with an assign statement.
 /// Once this is complete, Drop terminators in the MIR correspond to a call to the "drop glue" or
 /// "drop shim" for the type of the dropped place.
 ///
@@ -121,8 +120,7 @@ fn remove_dead_unwinds<'tcx>(
         .into_results_cursor(body);
     for (bb, bb_data) in body.basic_blocks.iter_enumerated() {
         let place = match bb_data.terminator().kind {
-            TerminatorKind::Drop { ref place, unwind: Some(_), .. }
-            | TerminatorKind::DropAndReplace { ref place, unwind: Some(_), .. } => {
+            TerminatorKind::Drop { ref place, unwind: UnwindAction::Cleanup(_), .. } => {
                 und.derefer(place.as_ref(), body).unwrap_or(*place)
             }
             _ => continue,
@@ -162,7 +160,7 @@ fn remove_dead_unwinds<'tcx>(
     let basic_blocks = body.basic_blocks.as_mut();
     for &bb in dead_unwinds.iter() {
         if let Some(unwind) = basic_blocks[bb].terminator_mut().unwind_mut() {
-            *unwind = None;
+            *unwind = UnwindAction::Unreachable;
         }
     }
 }
@@ -254,7 +252,7 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, 'tcx> {
         }
     }
 
-    fn field_subpath(&self, path: Self::Path, field: Field) -> Option<Self::Path> {
+    fn field_subpath(&self, path: Self::Path, field: FieldIdx) -> Option<Self::Path> {
         rustc_mir_dataflow::move_path_children_matching(self.ctxt.move_data(), path, |e| match e {
             ProjectionElem::Field(idx, _) => idx == field,
             _ => false,
@@ -343,8 +341,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
             }
             let terminator = data.terminator();
             let place = match terminator.kind {
-                TerminatorKind::Drop { ref place, .. }
-                | TerminatorKind::DropAndReplace { ref place, .. } => {
+                TerminatorKind::Drop { ref place, .. } => {
                     self.un_derefer.derefer(place.as_ref(), self.body).unwrap_or(*place)
                 }
                 _ => continue,
@@ -402,7 +399,6 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
             let loc = Location { block: bb, statement_index: data.statements.len() };
             let terminator = data.terminator();
 
-            let resume_block = self.patch.resume_block();
             match terminator.kind {
                 TerminatorKind::Drop { mut place, target, unwind } => {
                     if let Some(new_place) = self.un_derefer.derefer(place.as_ref(), self.body) {
@@ -411,124 +407,53 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
 
                     self.init_data.seek_before(loc);
                     match self.move_data().rev_lookup.find(place.as_ref()) {
-                        LookupResult::Exact(path) => elaborate_drop(
-                            &mut Elaborator { ctxt: self },
-                            terminator.source_info,
-                            place,
-                            path,
-                            target,
-                            if data.is_cleanup {
+                        LookupResult::Exact(path) => {
+                            let unwind = if data.is_cleanup {
                                 Unwind::InCleanup
                             } else {
-                                Unwind::To(Option::unwrap_or(unwind, resume_block))
-                            },
-                            bb,
-                        ),
+                                match unwind {
+                                    UnwindAction::Cleanup(cleanup) => Unwind::To(cleanup),
+                                    UnwindAction::Continue => Unwind::To(self.patch.resume_block()),
+                                    UnwindAction::Unreachable => {
+                                        Unwind::To(self.patch.unreachable_cleanup_block())
+                                    }
+                                    UnwindAction::Terminate => {
+                                        Unwind::To(self.patch.terminate_block())
+                                    }
+                                }
+                            };
+                            elaborate_drop(
+                                &mut Elaborator { ctxt: self },
+                                terminator.source_info,
+                                place,
+                                path,
+                                target,
+                                unwind,
+                                bb,
+                            )
+                        }
                         LookupResult::Parent(..) => {
-                            self.tcx.sess.delay_span_bug(
-                                terminator.source_info.span,
-                                &format!("drop of untracked value {:?}", bb),
-                            );
+                            if !matches!(
+                                terminator.source_info.span.desugaring_kind(),
+                                Some(DesugaringKind::Replace),
+                            ) {
+                                self.tcx.sess.delay_span_bug(
+                                    terminator.source_info.span,
+                                    &format!("drop of untracked value {:?}", bb),
+                                );
+                            }
+                            // A drop and replace behind a pointer/array/whatever.
+                            // The borrow checker requires that these locations are initialized before the assignment,
+                            // so we just leave an unconditional drop.
+                            assert!(!data.is_cleanup);
                         }
                     }
                 }
-                TerminatorKind::DropAndReplace { mut place, ref value, target, unwind } => {
-                    assert!(!data.is_cleanup);
-
-                    if let Some(new_place) = self.un_derefer.derefer(place.as_ref(), self.body) {
-                        place = new_place;
-                    }
-                    self.elaborate_replace(loc, place, value, target, unwind);
-                }
                 _ => continue,
             }
         }
     }
 
-    /// Elaborate a MIR `replace` terminator. This instruction
-    /// is not directly handled by codegen, and therefore
-    /// must be desugared.
-    ///
-    /// The desugaring drops the location if needed, and then writes
-    /// the value (including setting the drop flag) over it in *both* arms.
-    ///
-    /// The `replace` terminator can also be called on places that
-    /// are not tracked by elaboration (for example,
-    /// `replace x[i] <- tmp0`). The borrow checker requires that
-    /// these locations are initialized before the assignment,
-    /// so we just generate an unconditional drop.
-    fn elaborate_replace(
-        &mut self,
-        loc: Location,
-        place: Place<'tcx>,
-        value: &Operand<'tcx>,
-        target: BasicBlock,
-        unwind: Option<BasicBlock>,
-    ) {
-        let bb = loc.block;
-        let data = &self.body[bb];
-        let terminator = data.terminator();
-        assert!(!data.is_cleanup, "DropAndReplace in unwind path not supported");
-
-        let assign = Statement {
-            kind: StatementKind::Assign(Box::new((place, Rvalue::Use(value.clone())))),
-            source_info: terminator.source_info,
-        };
-
-        let unwind = unwind.unwrap_or_else(|| self.patch.resume_block());
-        let unwind = self.patch.new_block(BasicBlockData {
-            statements: vec![assign.clone()],
-            terminator: Some(Terminator {
-                kind: TerminatorKind::Goto { target: unwind },
-                ..*terminator
-            }),
-            is_cleanup: true,
-        });
-
-        let target = self.patch.new_block(BasicBlockData {
-            statements: vec![assign],
-            terminator: Some(Terminator { kind: TerminatorKind::Goto { target }, ..*terminator }),
-            is_cleanup: false,
-        });
-
-        match self.move_data().rev_lookup.find(place.as_ref()) {
-            LookupResult::Exact(path) => {
-                debug!("elaborate_drop_and_replace({:?}) - tracked {:?}", terminator, path);
-                self.init_data.seek_before(loc);
-                elaborate_drop(
-                    &mut Elaborator { ctxt: self },
-                    terminator.source_info,
-                    place,
-                    path,
-                    target,
-                    Unwind::To(unwind),
-                    bb,
-                );
-                on_all_children_bits(self.tcx, self.body, self.move_data(), path, |child| {
-                    self.set_drop_flag(
-                        Location { block: target, statement_index: 0 },
-                        child,
-                        DropFlagState::Present,
-                    );
-                    self.set_drop_flag(
-                        Location { block: unwind, statement_index: 0 },
-                        child,
-                        DropFlagState::Present,
-                    );
-                });
-            }
-            LookupResult::Parent(parent) => {
-                // drop and replace behind a pointer/array/whatever. The location
-                // must be initialized.
-                debug!("elaborate_drop_and_replace({:?}) - untracked {:?}", terminator, parent);
-                self.patch.patch_terminator(
-                    bb,
-                    TerminatorKind::Drop { place, target, unwind: Some(unwind) },
-                );
-            }
-        }
-    }
-
     fn constant_bool(&self, span: Span, val: bool) -> Rvalue<'tcx> {
         Rvalue::Use(Operand::Constant(Box::new(Constant {
             span,
@@ -560,7 +485,10 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
                 continue;
             }
             if let TerminatorKind::Call {
-                destination, target: Some(tgt), cleanup: Some(_), ..
+                destination,
+                target: Some(tgt),
+                unwind: UnwindAction::Cleanup(_),
+                ..
             } = data.terminator().kind
             {
                 assert!(!self.patch.is_patched(bb));
@@ -600,22 +528,12 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
             debug!("drop_flags_for_locs({:?})", data);
             for i in 0..(data.statements.len() + 1) {
                 debug!("drop_flag_for_locs: stmt {}", i);
-                let mut allow_initializations = true;
                 if i == data.statements.len() {
                     match data.terminator().kind {
                         TerminatorKind::Drop { .. } => {
                             // drop elaboration should handle that by itself
                             continue;
                         }
-                        TerminatorKind::DropAndReplace { .. } => {
-                            // this contains the move of the source and
-                            // the initialization of the destination. We
-                            // only want the former - the latter is handled
-                            // by the elaboration code and must be done
-                            // *after* the destination is dropped.
-                            assert!(self.patch.is_patched(bb));
-                            allow_initializations = false;
-                        }
                         TerminatorKind::Resume => {
                             // It is possible for `Resume` to be patched
                             // (in particular it can be patched to be replaced with
@@ -632,19 +550,19 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
                     self.body,
                     self.env,
                     loc,
-                    |path, ds| {
-                        if ds == DropFlagState::Absent || allow_initializations {
-                            self.set_drop_flag(loc, path, ds)
-                        }
-                    },
+                    |path, ds| self.set_drop_flag(loc, path, ds),
                 )
             }
 
             // There may be a critical edge after this call,
             // so mark the return as initialized *before* the
             // call.
-            if let TerminatorKind::Call { destination, target: Some(_), cleanup: None, .. } =
-                data.terminator().kind
+            if let TerminatorKind::Call {
+                destination,
+                target: Some(_),
+                unwind: UnwindAction::Continue | UnwindAction::Unreachable | UnwindAction::Terminate,
+                ..
+            } = data.terminator().kind
             {
                 assert!(!self.patch.is_patched(bb));
 
diff --git a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs
index e6546911a..c9b24adba 100644
--- a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs
+++ b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs
@@ -1,5 +1,6 @@
-use rustc_hir::def_id::{CrateNum, LocalDefId, LOCAL_CRATE};
+use rustc_hir::def_id::{LocalDefId, LOCAL_CRATE};
 use rustc_middle::mir::*;
+use rustc_middle::query::LocalCrate;
 use rustc_middle::ty::layout;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
@@ -121,9 +122,7 @@ fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool {
     tainted
 }
 
-fn required_panic_strategy(tcx: TyCtxt<'_>, cnum: CrateNum) -> Option<PanicStrategy> {
-    assert_eq!(cnum, LOCAL_CRATE);
-
+fn required_panic_strategy(tcx: TyCtxt<'_>, _: LocalCrate) -> Option<PanicStrategy> {
     if tcx.is_panic_runtime(LOCAL_CRATE) {
         return Some(tcx.sess.panic_strategy());
     }
diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs
index 66d32b954..8601c1b2d 100644
--- a/compiler/rustc_mir_transform/src/function_item_references.rs
+++ b/compiler/rustc_mir_transform/src/function_item_references.rs
@@ -34,7 +34,7 @@ impl<'tcx> Visitor<'tcx> for FunctionItemRefChecker<'_, 'tcx> {
             args,
             destination: _,
             target: _,
-            cleanup: _,
+            unwind: _,
             from_hir_call: _,
             fn_span: _,
         } = &terminator.kind
diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs
index 2e97312ee..4c4423721 100644
--- a/compiler/rustc_mir_transform/src/generator.rs
+++ b/compiler/rustc_mir_transform/src/generator.rs
@@ -70,10 +70,10 @@ use rustc_mir_dataflow::impls::{
 };
 use rustc_mir_dataflow::storage::always_storage_live_locals;
 use rustc_mir_dataflow::{self, Analysis};
-use rustc_span::def_id::DefId;
+use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 use rustc_target::spec::PanicStrategy;
 use std::{iter, ops};
 
@@ -162,9 +162,10 @@ impl<'tcx> MutVisitor<'tcx> for PinArgVisitor<'tcx> {
                 place,
                 Place {
                     local: SELF_ARG,
-                    projection: self
-                        .tcx()
-                        .mk_place_elems(&[ProjectionElem::Field(Field::new(0), self.ref_gen_ty)]),
+                    projection: self.tcx().mk_place_elems(&[ProjectionElem::Field(
+                        FieldIdx::new(0),
+                        self.ref_gen_ty,
+                    )]),
                 },
                 self.tcx,
             );
@@ -273,7 +274,7 @@ impl<'tcx> TransformVisitor<'tcx> {
             statements.push(Statement {
                 kind: StatementKind::Assign(Box::new((
                     Place::return_place(),
-                    Rvalue::Aggregate(Box::new(kind), vec![]),
+                    Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
                 ))),
                 source_info,
             });
@@ -286,7 +287,7 @@ impl<'tcx> TransformVisitor<'tcx> {
         statements.push(Statement {
             kind: StatementKind::Assign(Box::new((
                 Place::return_place(),
-                Rvalue::Aggregate(Box::new(kind), vec![val]),
+                Rvalue::Aggregate(Box::new(kind), [val].into()),
             ))),
             source_info,
         });
@@ -297,7 +298,7 @@ impl<'tcx> TransformVisitor<'tcx> {
         let self_place = Place::from(SELF_ARG);
         let base = self.tcx.mk_place_downcast_unnamed(self_place, variant_index);
         let mut projection = base.projection.to_vec();
-        projection.push(ProjectionElem::Field(Field::new(idx), ty));
+        projection.push(ProjectionElem::Field(FieldIdx::new(idx), ty));
 
         Place { local: base.local, projection: self.tcx.mk_place_elems(&projection) }
     }
@@ -924,13 +925,19 @@ fn compute_layout<'tcx>(
         debug!(?decl);
 
         let ignore_for_traits = if tcx.sess.opts.unstable_opts.drop_tracking_mir {
+            // Do not `assert_crate_local` here, as post-borrowck cleanup may have already cleared
+            // the information. This is alright, since `ignore_for_traits` is only relevant when
+            // this code runs on pre-cleanup MIR, and `ignore_for_traits = false` is the safer
+            // default.
             match decl.local_info {
                 // Do not include raw pointers created from accessing `static` items, as those could
                 // well be re-created by another access to the same static.
-                Some(box LocalInfo::StaticRef { is_thread_local, .. }) => !is_thread_local,
+                ClearCrossCrate::Set(box LocalInfo::StaticRef { is_thread_local, .. }) => {
+                    !is_thread_local
+                }
                 // Fake borrows are only read by fake reads, so do not have any reality in
                 // post-analysis MIR.
-                Some(box LocalInfo::FakeBorrow) => true,
+                ClearCrossCrate::Set(box LocalInfo::FakeBorrow) => true,
                 _ => false,
             }
         } else {
@@ -961,7 +968,7 @@ fn compute_layout<'tcx>(
 
     // Build the generator variant field list.
     // Create a map from local indices to generator struct indices.
-    let mut variant_fields: IndexVec<VariantIdx, IndexVec<Field, GeneratorSavedLocal>> =
+    let mut variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, GeneratorSavedLocal>> =
         iter::repeat(IndexVec::new()).take(RESERVED_VARIANTS).collect();
     let mut remap = FxHashMap::default();
     for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() {
@@ -1053,7 +1060,12 @@ fn elaborate_generator_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         let unwind = if block_data.is_cleanup {
             Unwind::InCleanup
         } else {
-            Unwind::To(unwind.unwrap_or_else(|| elaborator.patch.resume_block()))
+            Unwind::To(match *unwind {
+                UnwindAction::Cleanup(tgt) => tgt,
+                UnwindAction::Continue => elaborator.patch.resume_block(),
+                UnwindAction::Unreachable => elaborator.patch.unreachable_cleanup_block(),
+                UnwindAction::Terminate => elaborator.patch.terminate_block(),
+            })
         };
         elaborate_drop(
             &mut elaborator,
@@ -1140,7 +1152,7 @@ fn insert_panic_block<'tcx>(
         expected: true,
         msg: message,
         target: assert_block,
-        cleanup: None,
+        unwind: UnwindAction::Continue,
     };
 
     let source_info = SourceInfo::outermost(body.span);
@@ -1182,7 +1194,7 @@ fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool {
             // These never unwind.
             TerminatorKind::Goto { .. }
             | TerminatorKind::SwitchInt { .. }
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::GeneratorDrop
@@ -1199,7 +1211,6 @@ fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool {
 
             // These may unwind.
             TerminatorKind::Drop { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::Call { .. }
             | TerminatorKind::InlineAsm { .. }
             | TerminatorKind::Assert { .. } => return true,
@@ -1242,8 +1253,8 @@ fn create_generator_resume_function<'tcx>(
             } else if !block.is_cleanup {
                 // Any terminators that *can* unwind but don't have an unwind target set are also
                 // pointed at our poisoning block (unless they're part of the cleanup path).
-                if let Some(unwind @ None) = block.terminator_mut().unwind_mut() {
-                    *unwind = Some(poison_block);
+                if let Some(unwind @ UnwindAction::Continue) = block.terminator_mut().unwind_mut() {
+                    *unwind = UnwindAction::Cleanup(poison_block);
                 }
             }
         }
@@ -1288,8 +1299,11 @@ fn create_generator_resume_function<'tcx>(
 fn insert_clean_drop(body: &mut Body<'_>) -> BasicBlock {
     let return_block = insert_term_block(body, TerminatorKind::Return);
 
-    let term =
-        TerminatorKind::Drop { place: Place::from(SELF_ARG), target: return_block, unwind: None };
+    let term = TerminatorKind::Drop {
+        place: Place::from(SELF_ARG),
+        target: return_block,
+        unwind: UnwindAction::Continue,
+    };
     let source_info = SourceInfo::outermost(body.span);
 
     // Create a block to destroy an unresumed generators. This can only destroy upvars.
@@ -1381,10 +1395,9 @@ fn create_cases<'tcx>(
 #[instrument(level = "debug", skip(tcx), ret)]
 pub(crate) fn mir_generator_witnesses<'tcx>(
     tcx: TyCtxt<'tcx>,
-    def_id: DefId,
+    def_id: LocalDefId,
 ) -> GeneratorLayout<'tcx> {
     assert!(tcx.sess.opts.unstable_opts.drop_tracking_mir);
-    let def_id = def_id.expect_local();
 
     let (body, _) = tcx.mir_promoted(ty::WithOptConstParam::unknown(def_id));
     let body = body.borrow();
@@ -1543,6 +1556,13 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         body.arg_count = 2; // self, resume arg
         body.spread_arg = None;
 
+        // The original arguments to the function are no longer arguments, mark them as such.
+        // Otherwise they'll conflict with our new arguments, which although they don't have
+        // argument_index set, will get emitted as unnamed arguments.
+        for var in &mut body.var_debug_info {
+            var.argument_index = None;
+        }
+
         body.generator.as_mut().unwrap().yield_ty = None;
         body.generator.as_mut().unwrap().generator_layout = Some(layout);
 
@@ -1648,6 +1668,7 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> {
             | StatementKind::StorageDead(_)
             | StatementKind::Retag(..)
             | StatementKind::AscribeUserType(..)
+            | StatementKind::PlaceMention(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
             | StatementKind::ConstEvalCounter
@@ -1664,7 +1685,7 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> {
                 args,
                 destination,
                 target: Some(_),
-                cleanup: _,
+                unwind: _,
                 from_hir_call: _,
                 fn_span: _,
             } => {
@@ -1687,11 +1708,10 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> {
             | TerminatorKind::Goto { .. }
             | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::Drop { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::Assert { .. }
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::FalseEdge { .. }
@@ -1872,12 +1892,14 @@ fn check_must_not_suspend_def(
     data: SuspendCheckData<'_>,
 ) -> bool {
     if let Some(attr) = tcx.get_attr(def_id, sym::must_not_suspend) {
-        let msg = format!(
-            "{}`{}`{} held across a suspend point, but should not be",
-            data.descr_pre,
-            tcx.def_path_str(def_id),
-            data.descr_post,
-        );
+        let msg = rustc_errors::DelayDm(|| {
+            format!(
+                "{}`{}`{} held across a suspend point, but should not be",
+                data.descr_pre,
+                tcx.def_path_str(def_id),
+                data.descr_post,
+            )
+        });
         tcx.struct_span_lint_hir(
             rustc_session::lint::builtin::MUST_NOT_SUSPEND,
             hir_id,
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 6e6d6566f..f0cb317f4 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -10,7 +10,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
 use rustc_session::config::OptLevel;
 use rustc_span::{hygiene::ExpnKind, ExpnData, LocalExpnId, Span};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, FIRST_VARIANT};
 use rustc_target::spec::abi::Abi;
 
 use crate::simplify::{remove_dead_blocks, CfgSimplifier};
@@ -270,7 +270,9 @@ impl<'tcx> Inliner<'tcx> {
             | InstanceDef::FnPtrShim(..)
             | InstanceDef::ClosureOnceShim { .. }
             | InstanceDef::DropGlue(..)
-            | InstanceDef::CloneShim(..) => return Ok(()),
+            | InstanceDef::CloneShim(..)
+            | InstanceDef::ThreadLocalShim(..)
+            | InstanceDef::FnPtrAddrShim(..) => return Ok(()),
         }
 
         if self.tcx.is_constructor(callee_def_id) {
@@ -424,13 +426,6 @@ impl<'tcx> Inliner<'tcx> {
         debug!("    final inline threshold = {}", threshold);
 
         // FIXME: Give a bonus to functions with only a single caller
-        let diverges = matches!(
-            callee_body.basic_blocks[START_BLOCK].terminator().kind,
-            TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. }
-        );
-        if diverges && !matches!(callee_attrs.inline, InlineAttr::Always) {
-            return Err("callee diverges unconditionally");
-        }
 
         let mut checker = CostChecker {
             tcx: self.tcx,
@@ -453,14 +448,12 @@ impl<'tcx> Inliner<'tcx> {
             checker.visit_basic_block_data(bb, blk);
 
             let term = blk.terminator();
-            if let TerminatorKind::Drop { ref place, target, unwind }
-            | TerminatorKind::DropAndReplace { ref place, target, unwind, .. } = term.kind
-            {
+            if let TerminatorKind::Drop { ref place, target, unwind } = term.kind {
                 work_list.push(target);
 
                 // If the place doesn't actually need dropping, treat it like a regular goto.
                 let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty);
-                if ty.needs_drop(tcx, self.param_env) && let Some(unwind) = unwind {
+                if ty.needs_drop(tcx, self.param_env) && let UnwindAction::Cleanup(unwind) = unwind {
                         work_list.push(unwind);
                     }
             } else if callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set
@@ -507,7 +500,7 @@ impl<'tcx> Inliner<'tcx> {
     ) {
         let terminator = caller_body[callsite.block].terminator.take().unwrap();
         match terminator.kind {
-            TerminatorKind::Call { args, destination, cleanup, .. } => {
+            TerminatorKind::Call { args, destination, unwind, .. } => {
                 // If the call is something like `a[*i] = f(i)`, where
                 // `i : &mut usize`, then just duplicating the `a[*i]`
                 // Place could result in two different locations if `f`
@@ -578,7 +571,7 @@ impl<'tcx> Inliner<'tcx> {
                     destination: destination_local,
                     callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(),
                     callsite,
-                    cleanup_block: cleanup,
+                    cleanup_block: unwind,
                     in_cleanup_block: false,
                     tcx: self.tcx,
                     expn_data,
@@ -708,7 +701,7 @@ impl<'tcx> Inliner<'tcx> {
             // The `tmp0`, `tmp1`, and `tmp2` in our example above.
             let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| {
                 // This is e.g., `tuple_tmp.0` in our example above.
-                let tuple_field = Operand::Move(tcx.mk_place_field(tuple, Field::new(i), ty));
+                let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty));
 
                 // Spill to a local to make e.g., `tmp0`.
                 self.create_temp_if_necessary(tuple_field, callsite, caller_body)
@@ -815,20 +808,19 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
         let tcx = self.tcx;
         match terminator.kind {
-            TerminatorKind::Drop { ref place, unwind, .. }
-            | TerminatorKind::DropAndReplace { ref place, unwind, .. } => {
+            TerminatorKind::Drop { ref place, unwind, .. } => {
                 // If the place doesn't actually need dropping, treat it like a regular goto.
                 let ty = self.instance.subst_mir(tcx, &place.ty(self.callee_body, tcx).ty);
                 if ty.needs_drop(tcx, self.param_env) {
                     self.cost += CALL_PENALTY;
-                    if unwind.is_some() {
+                    if let UnwindAction::Cleanup(_) = unwind {
                         self.cost += LANDINGPAD_PENALTY;
                     }
                 } else {
                     self.cost += INSTR_COST;
                 }
             }
-            TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => {
+            TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => {
                 let fn_ty = self.instance.subst_mir(tcx, &f.literal.ty());
                 self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) {
                     // Don't give intrinsics the extra penalty for calls
@@ -836,20 +828,20 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
                 } else {
                     CALL_PENALTY
                 };
-                if cleanup.is_some() {
+                if let UnwindAction::Cleanup(_) = unwind {
                     self.cost += LANDINGPAD_PENALTY;
                 }
             }
-            TerminatorKind::Assert { cleanup, .. } => {
+            TerminatorKind::Assert { unwind, .. } => {
                 self.cost += CALL_PENALTY;
-                if cleanup.is_some() {
+                if let UnwindAction::Cleanup(_) = unwind {
                     self.cost += LANDINGPAD_PENALTY;
                 }
             }
             TerminatorKind::Resume => self.cost += RESUME_PENALTY,
-            TerminatorKind::InlineAsm { cleanup, .. } => {
+            TerminatorKind::InlineAsm { unwind, .. } => {
                 self.cost += INSTR_COST;
-                if cleanup.is_some() {
+                if let UnwindAction::Cleanup(_) = unwind {
                     self.cost += LANDINGPAD_PENALTY;
                 }
             }
@@ -914,8 +906,8 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
                     check_equal(self, *f_ty);
                 }
                 ty::Adt(adt_def, substs) => {
-                    let var = parent_ty.variant_index.unwrap_or(VariantIdx::from_u32(0));
-                    let Some(field) = adt_def.variant(var).fields.get(f.as_usize()) else {
+                    let var = parent_ty.variant_index.unwrap_or(FIRST_VARIANT);
+                    let Some(field) = adt_def.variant(var).fields.get(f) else {
                         self.validation = Err("malformed MIR");
                         return;
                     };
@@ -987,7 +979,7 @@ struct Integrator<'a, 'tcx> {
     destination: Local,
     callsite_scope: SourceScopeData<'tcx>,
     callsite: &'a CallSite<'tcx>,
-    cleanup_block: Option<BasicBlock>,
+    cleanup_block: UnwindAction,
     in_cleanup_block: bool,
     tcx: TyCtxt<'tcx>,
     expn_data: LocalExpnId,
@@ -1022,18 +1014,21 @@ impl Integrator<'_, '_> {
         new
     }
 
-    fn map_unwind(&self, unwind: Option<BasicBlock>) -> Option<BasicBlock> {
+    fn map_unwind(&self, unwind: UnwindAction) -> UnwindAction {
         if self.in_cleanup_block {
-            if unwind.is_some() {
-                bug!("cleanup on cleanup block");
+            match unwind {
+                UnwindAction::Cleanup(_) | UnwindAction::Continue => {
+                    bug!("cleanup on cleanup block");
+                }
+                UnwindAction::Unreachable | UnwindAction::Terminate => return unwind,
             }
-            return unwind;
         }
 
         match unwind {
-            Some(target) => Some(self.map_block(target)),
+            UnwindAction::Unreachable | UnwindAction::Terminate => unwind,
+            UnwindAction::Cleanup(target) => UnwindAction::Cleanup(self.map_block(target)),
             // Add an unwind edge to the original call's cleanup block
-            None => self.cleanup_block,
+            UnwindAction::Continue => self.cleanup_block,
         }
     }
 }
@@ -1120,20 +1115,19 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
                     *tgt = self.map_block(*tgt);
                 }
             }
-            TerminatorKind::Drop { ref mut target, ref mut unwind, .. }
-            | TerminatorKind::DropAndReplace { ref mut target, ref mut unwind, .. } => {
+            TerminatorKind::Drop { ref mut target, ref mut unwind, .. } => {
                 *target = self.map_block(*target);
                 *unwind = self.map_unwind(*unwind);
             }
-            TerminatorKind::Call { ref mut target, ref mut cleanup, .. } => {
+            TerminatorKind::Call { ref mut target, ref mut unwind, .. } => {
                 if let Some(ref mut tgt) = *target {
                     *tgt = self.map_block(*tgt);
                 }
-                *cleanup = self.map_unwind(*cleanup);
+                *unwind = self.map_unwind(*unwind);
             }
-            TerminatorKind::Assert { ref mut target, ref mut cleanup, .. } => {
+            TerminatorKind::Assert { ref mut target, ref mut unwind, .. } => {
                 *target = self.map_block(*target);
-                *cleanup = self.map_unwind(*cleanup);
+                *unwind = self.map_unwind(*unwind);
             }
             TerminatorKind::Return => {
                 terminator.kind = if let Some(tgt) = self.callsite.target {
@@ -1143,11 +1137,14 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
                 }
             }
             TerminatorKind::Resume => {
-                if let Some(tgt) = self.cleanup_block {
-                    terminator.kind = TerminatorKind::Goto { target: tgt }
-                }
+                terminator.kind = match self.cleanup_block {
+                    UnwindAction::Cleanup(tgt) => TerminatorKind::Goto { target: tgt },
+                    UnwindAction::Continue => TerminatorKind::Resume,
+                    UnwindAction::Unreachable => TerminatorKind::Unreachable,
+                    UnwindAction::Terminate => TerminatorKind::Terminate,
+                };
             }
-            TerminatorKind::Abort => {}
+            TerminatorKind::Terminate => {}
             TerminatorKind::Unreachable => {}
             TerminatorKind::FalseEdge { ref mut real_target, ref mut imaginary_target } => {
                 *real_target = self.map_block(*real_target);
@@ -1158,11 +1155,11 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
             {
                 bug!("False unwinds should have been removed before inlining")
             }
-            TerminatorKind::InlineAsm { ref mut destination, ref mut cleanup, .. } => {
+            TerminatorKind::InlineAsm { ref mut destination, ref mut unwind, .. } => {
                 if let Some(ref mut tgt) = *destination {
                     *tgt = self.map_block(*tgt);
                 }
-                *cleanup = self.map_unwind(*cleanup);
+                *unwind = self.map_unwind(*unwind);
             }
         }
     }
diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs
index 792457c80..8aa3c23d0 100644
--- a/compiler/rustc_mir_transform/src/inline/cycle.rs
+++ b/compiler/rustc_mir_transform/src/inline/cycle.rs
@@ -83,7 +83,11 @@ pub(crate) fn mir_callgraph_reachable<'tcx>(
                 | InstanceDef::ReifyShim(_)
                 | InstanceDef::FnPtrShim(..)
                 | InstanceDef::ClosureOnceShim { .. }
+                | InstanceDef::ThreadLocalShim { .. }
                 | InstanceDef::CloneShim(..) => {}
+
+                // This shim does not call any other functions, thus there can be no recursion.
+                InstanceDef::FnPtrAddrShim(..) => continue,
                 InstanceDef::DropGlue(..) => {
                     // FIXME: A not fully substituted drop shim can cause ICEs if one attempts to
                     // have its MIR built. Likely oli-obk just screwed up the `ParamEnv`s, so this
diff --git a/compiler/rustc_mir_transform/src/instcombine.rs b/compiler/rustc_mir_transform/src/instcombine.rs
index 4182da195..3d06a0a49 100644
--- a/compiler/rustc_mir_transform/src/instcombine.rs
+++ b/compiler/rustc_mir_transform/src/instcombine.rs
@@ -3,12 +3,14 @@
 use crate::MirPass;
 use rustc_hir::Mutability;
 use rustc_middle::mir::{
-    BinOp, Body, Constant, ConstantKind, LocalDecls, Operand, Place, ProjectionElem, Rvalue,
-    SourceInfo, Statement, StatementKind, Terminator, TerminatorKind, UnOp,
+    BinOp, Body, CastKind, Constant, ConstantKind, LocalDecls, Operand, Place, ProjectionElem,
+    Rvalue, SourceInfo, Statement, StatementKind, SwitchTargets, Terminator, TerminatorKind, UnOp,
 };
 use rustc_middle::ty::layout::ValidityRequirement;
+use rustc_middle::ty::util::IntTypeExt;
 use rustc_middle::ty::{self, ParamEnv, SubstsRef, Ty, TyCtxt};
 use rustc_span::symbol::Symbol;
+use rustc_target::abi::FieldIdx;
 
 pub struct InstCombine;
 
@@ -44,6 +46,7 @@ impl<'tcx> MirPass<'tcx> for InstCombine {
                 &mut block.terminator.as_mut().unwrap(),
                 &mut block.statements,
             );
+            ctx.combine_duplicate_switch_targets(&mut block.terminator.as_mut().unwrap());
         }
     }
 }
@@ -144,9 +147,53 @@ impl<'tcx> InstCombineContext<'tcx, '_> {
     }
 
     fn combine_cast(&self, _source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
-        if let Rvalue::Cast(_kind, operand, ty) = rvalue {
-            if operand.ty(self.local_decls, self.tcx) == *ty {
+        if let Rvalue::Cast(kind, operand, cast_ty) = rvalue {
+            let operand_ty = operand.ty(self.local_decls, self.tcx);
+            if operand_ty == *cast_ty {
                 *rvalue = Rvalue::Use(operand.clone());
+            } else if *kind == CastKind::Transmute {
+                // Transmuting an integer to another integer is just a signedness cast
+                if let (ty::Int(int), ty::Uint(uint)) | (ty::Uint(uint), ty::Int(int)) = (operand_ty.kind(), cast_ty.kind())
+                    && int.bit_width() == uint.bit_width()
+                {
+                    // The width check isn't strictly necessary, as different widths
+                    // are UB and thus we'd be allowed to turn it into a cast anyway.
+                    // But let's keep the UB around for codegen to exploit later.
+                    // (If `CastKind::Transmute` ever becomes *not* UB for mismatched sizes,
+                    // then the width check is necessary for big-endian correctness.)
+                    *kind = CastKind::IntToInt;
+                    return;
+                }
+
+                // Transmuting a fieldless enum to its repr is a discriminant read
+                if let ty::Adt(adt_def, ..) = operand_ty.kind()
+                    && adt_def.is_enum()
+                    && adt_def.is_payloadfree()
+                    && let Some(place) = operand.place()
+                    && let Some(repr_int) = adt_def.repr().int
+                    && repr_int.to_ty(self.tcx) == *cast_ty
+                {
+                    *rvalue = Rvalue::Discriminant(place);
+                    return;
+                }
+
+                // Transmuting a transparent struct/union to a field's type is a projection
+                if let ty::Adt(adt_def, substs) = operand_ty.kind()
+                    && adt_def.repr().transparent()
+                    && (adt_def.is_struct() || adt_def.is_union())
+                    && let Some(place) = operand.place()
+                {
+                    let variant = adt_def.non_enum_variant();
+                    for (i, field) in variant.fields.iter().enumerate() {
+                        let field_ty = field.ty(self.tcx, substs);
+                        if field_ty == *cast_ty {
+                            let place = place.project_deeper(&[ProjectionElem::Field(FieldIdx::from_usize(i), *cast_ty)], self.tcx);
+                            let operand = if operand.is_move() { Operand::Move(place) } else { Operand::Copy(place) };
+                            *rvalue = Rvalue::Use(operand);
+                            return;
+                        }
+                    }
+                }
             }
         }
     }
@@ -217,6 +264,19 @@ impl<'tcx> InstCombineContext<'tcx, '_> {
         terminator.kind = TerminatorKind::Goto { target: destination_block };
     }
 
+    fn combine_duplicate_switch_targets(&self, terminator: &mut Terminator<'tcx>) {
+        let TerminatorKind::SwitchInt { targets, .. } = &mut terminator.kind
+        else { return };
+
+        let otherwise = targets.otherwise();
+        if targets.iter().any(|t| t.1 == otherwise) {
+            *targets = SwitchTargets::new(
+                targets.iter().filter(|t| t.1 != otherwise),
+                targets.otherwise(),
+            );
+        }
+    }
+
     fn combine_intrinsic_assert(
         &self,
         terminator: &mut Terminator<'tcx>,
diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs
index 89e0a007d..9447a2ff0 100644
--- a/compiler/rustc_mir_transform/src/large_enums.rs
+++ b/compiler/rustc_mir_transform/src/large_enums.rs
@@ -158,10 +158,12 @@ impl EnumSizeOpt {
                             tmp_ty,
                         ),
                     };
-                    let rval = Rvalue::Use(Operand::Constant(box (constant_vals)));
+                    let rval = Rvalue::Use(Operand::Constant(Box::new(constant_vals)));
 
-                    let const_assign =
-                        Statement { source_info, kind: StatementKind::Assign(box (place, rval)) };
+                    let const_assign = Statement {
+                        source_info,
+                        kind: StatementKind::Assign(Box::new((place, rval))),
+                    };
 
                     let discr_place = Place::from(
                         local_decls
@@ -170,7 +172,10 @@ impl EnumSizeOpt {
 
                     let store_discr = Statement {
                         source_info,
-                        kind: StatementKind::Assign(box (discr_place, Rvalue::Discriminant(*rhs))),
+                        kind: StatementKind::Assign(Box::new((
+                            discr_place,
+                            Rvalue::Discriminant(*rhs),
+                        ))),
                     };
 
                     let discr_cast_place =
@@ -178,14 +183,14 @@ impl EnumSizeOpt {
 
                     let cast_discr = Statement {
                         source_info,
-                        kind: StatementKind::Assign(box (
+                        kind: StatementKind::Assign(Box::new((
                             discr_cast_place,
                             Rvalue::Cast(
                                 CastKind::IntToInt,
                                 Operand::Copy(discr_place),
                                 tcx.types.usize,
                             ),
-                        )),
+                        ))),
                     };
 
                     let size_place =
@@ -193,14 +198,14 @@ impl EnumSizeOpt {
 
                     let store_size = Statement {
                         source_info,
-                        kind: StatementKind::Assign(box (
+                        kind: StatementKind::Assign(Box::new((
                             size_place,
                             Rvalue::Use(Operand::Copy(Place {
                                 local: size_array_local,
                                 projection: tcx
                                     .mk_place_elems(&[PlaceElem::Index(discr_cast_place.local)]),
                             })),
-                        )),
+                        ))),
                     };
 
                     let dst =
@@ -208,10 +213,10 @@ impl EnumSizeOpt {
 
                     let dst_ptr = Statement {
                         source_info,
-                        kind: StatementKind::Assign(box (
+                        kind: StatementKind::Assign(Box::new((
                             dst,
                             Rvalue::AddressOf(Mutability::Mut, *lhs),
-                        )),
+                        ))),
                     };
 
                     let dst_cast_ty = tcx.mk_mut_ptr(tcx.types.u8);
@@ -220,10 +225,10 @@ impl EnumSizeOpt {
 
                     let dst_cast = Statement {
                         source_info,
-                        kind: StatementKind::Assign(box (
+                        kind: StatementKind::Assign(Box::new((
                             dst_cast_place,
                             Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(dst), dst_cast_ty),
-                        )),
+                        ))),
                     };
 
                     let src =
@@ -231,10 +236,10 @@ impl EnumSizeOpt {
 
                     let src_ptr = Statement {
                         source_info,
-                        kind: StatementKind::Assign(box (
+                        kind: StatementKind::Assign(Box::new((
                             src,
                             Rvalue::AddressOf(Mutability::Not, *rhs),
-                        )),
+                        ))),
                     };
 
                     let src_cast_ty = tcx.mk_imm_ptr(tcx.types.u8);
@@ -243,24 +248,24 @@ impl EnumSizeOpt {
 
                     let src_cast = Statement {
                         source_info,
-                        kind: StatementKind::Assign(box (
+                        kind: StatementKind::Assign(Box::new((
                             src_cast_place,
                             Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(src), src_cast_ty),
-                        )),
+                        ))),
                     };
 
                     let deinit_old =
-                        Statement { source_info, kind: StatementKind::Deinit(box dst) };
+                        Statement { source_info, kind: StatementKind::Deinit(Box::new(dst)) };
 
                     let copy_bytes = Statement {
                         source_info,
-                        kind: StatementKind::Intrinsic(
-                            box NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
+                        kind: StatementKind::Intrinsic(Box::new(
+                            NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
                                 src: Operand::Copy(src_cast_place),
                                 dst: Operand::Copy(dst_cast_place),
                                 count: Operand::Copy(size_place),
                             }),
-                        ),
+                        )),
                     };
 
                     let store_dead = Statement {
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index cdd28ae0c..fc12d423c 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -1,12 +1,10 @@
 #![allow(rustc::potential_query_instability)]
 #![feature(box_patterns)]
 #![feature(drain_filter)]
-#![feature(box_syntax)]
 #![feature(let_chains)]
 #![feature(map_try_insert)]
 #![feature(min_specialization)]
 #![feature(never_type)]
-#![feature(once_cell)]
 #![feature(option_get_or_insert_default)]
 #![feature(trusted_step)]
 #![feature(try_blocks)]
@@ -30,9 +28,9 @@ use rustc_hir::intravisit::{self, Visitor};
 use rustc_index::vec::IndexVec;
 use rustc_middle::mir::visit::Visitor as _;
 use rustc_middle::mir::{
-    traversal, AnalysisPhase, Body, ConstQualifs, Constant, LocalDecl, MirPass, MirPhase, Operand,
-    Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, Statement, StatementKind,
-    TerminatorKind,
+    traversal, AnalysisPhase, Body, ClearCrossCrate, ConstQualifs, Constant, LocalDecl, MirPass,
+    MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo,
+    Statement, StatementKind, TerminatorKind,
 };
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
@@ -92,6 +90,7 @@ mod separate_const_switch;
 mod shim;
 mod ssa;
 // This pass is public to allow external drivers to perform MIR cleanup
+mod check_alignment;
 pub mod simplify;
 mod simplify_branches;
 mod simplify_comparison_integral;
@@ -113,7 +112,6 @@ pub fn provide(providers: &mut Providers) {
         mir_keys,
         mir_const,
         mir_const_qualif: |tcx, def_id| {
-            let def_id = def_id.expect_local();
             if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) {
                 tcx.mir_const_qualif_const_arg(def)
             } else {
@@ -134,7 +132,6 @@ pub fn provide(providers: &mut Providers) {
         mir_callgraph_reachable: inline::cycle::mir_callgraph_reachable,
         mir_inliner_callees: inline::cycle::mir_inliner_callees,
         promoted_mir: |tcx, def_id| {
-            let def_id = def_id.expect_local();
             if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) {
                 tcx.promoted_mir_of_const_arg(def)
             } else {
@@ -162,7 +159,7 @@ fn remap_mir_for_const_eval_select<'tcx>(
                 ref mut args,
                 destination,
                 target,
-                cleanup,
+                unwind,
                 fn_span,
                 ..
             } if let ty::FnDef(def_id, _) = *literal.ty().kind()
@@ -199,7 +196,7 @@ fn remap_mir_for_const_eval_select<'tcx>(
                     };
                     method(place)
                 }).collect();
-                terminator.kind = TerminatorKind::Call { func, args: arguments, destination, target, cleanup, from_hir_call: false, fn_span };
+                terminator.kind = TerminatorKind::Call { func, args: arguments, destination, target, unwind, from_hir_call: false, fn_span };
             }
             _ => {}
         }
@@ -207,8 +204,7 @@ fn remap_mir_for_const_eval_select<'tcx>(
     body
 }
 
-fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
-    let def_id = def_id.expect_local();
+fn is_mir_available(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     tcx.mir_keys(()).contains(&def_id)
 }
 
@@ -278,14 +274,14 @@ fn mir_const(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> &Steal<
     // Unsafety check uses the raw mir, so make sure it is run.
     if !tcx.sess.opts.unstable_opts.thir_unsafeck {
         if let Some(param_did) = def.const_param_did {
-            tcx.ensure().unsafety_check_result_for_const_arg((def.did, param_did));
+            tcx.ensure_with_value().unsafety_check_result_for_const_arg((def.did, param_did));
         } else {
-            tcx.ensure().unsafety_check_result(def.did);
+            tcx.ensure_with_value().unsafety_check_result(def.did);
         }
     }
 
     // has_ffi_unwind_calls query uses the raw mir, so make sure it is run.
-    tcx.ensure().has_ffi_unwind_calls(def.did);
+    tcx.ensure_with_value().has_ffi_unwind_calls(def.did);
 
     let mut body = tcx.mir_built(def).steal();
 
@@ -351,12 +347,11 @@ fn mir_promoted(
 }
 
 /// Compute the MIR that is used during CTFE (and thus has no optimizations run on it)
-fn mir_for_ctfe(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> {
-    let did = def_id.expect_local();
-    if let Some(def) = ty::WithOptConstParam::try_lookup(did, tcx) {
+fn mir_for_ctfe(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &Body<'_> {
+    if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) {
         tcx.mir_for_ctfe_of_const_arg(def)
     } else {
-        tcx.arena.alloc(inner_mir_for_ctfe(tcx, ty::WithOptConstParam::unknown(did)))
+        tcx.arena.alloc(inner_mir_for_ctfe(tcx, ty::WithOptConstParam::unknown(def_id)))
     }
 }
 
@@ -416,8 +411,6 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -
 
     pm::run_passes(tcx, &mut body, &[&ctfe_limit::CtfeLimit], None);
 
-    debug_assert!(!body.has_free_regions(), "Free regions in MIR for CTFE");
-
     body
 }
 
@@ -435,7 +428,7 @@ fn mir_drops_elaborated_and_const_checked(
     if tcx.sess.opts.unstable_opts.drop_tracking_mir
         && let DefKind::Generator = tcx.def_kind(def.did)
     {
-        tcx.ensure().mir_generator_witnesses(def.did);
+        tcx.ensure_with_value().mir_generator_witnesses(def.did);
     }
     let mir_borrowck = tcx.mir_borrowck_opt_const_arg(def);
 
@@ -446,7 +439,7 @@ fn mir_drops_elaborated_and_const_checked(
 
         // Do not compute the mir call graph without said call graph actually being used.
         if inline::Inline.is_enabled(&tcx.sess) {
-            let _ = tcx.mir_inliner_callees(ty::InstanceDef::Item(def));
+            tcx.ensure_with_value().mir_inliner_callees(ty::InstanceDef::Item(def));
         }
     }
 
@@ -535,6 +528,12 @@ fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         &[&lower_intrinsics::LowerIntrinsics, &simplify::SimplifyCfg::new("elaborate-drops")];
 
     pm::run_passes(tcx, body, passes, Some(MirPhase::Runtime(RuntimePhase::PostCleanup)));
+
+    // Clear this by anticipation. Optimizations and runtime MIR have no reason to look
+    // into this information, which is meant for borrowck diagnostics.
+    for decl in &mut body.local_decls {
+        decl.local_info = ClearCrossCrate::Clear;
+    }
 }
 
 fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -547,6 +546,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         tcx,
         body,
         &[
+            &check_alignment::CheckAlignment,
             &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode.
             &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first
             &unreachable_prop::UnreachablePropagation,
@@ -566,8 +566,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &separate_const_switch::SeparateConstSwitch,
             &simplify::SimplifyLocals::new("before-const-prop"),
             &copy_prop::CopyProp,
-            //
-            // FIXME(#70073): This pass is responsible for both optimization as well as some lints.
             &const_prop::ConstProp,
             &dataflow_const_prop::DataflowConstProp,
             //
@@ -596,8 +594,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 }
 
 /// Optimize the MIR and prepare it for codegen.
-fn optimized_mir(tcx: TyCtxt<'_>, did: DefId) -> &Body<'_> {
-    let did = did.expect_local();
+fn optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> &Body<'_> {
     assert_eq!(ty::WithOptConstParam::try_lookup(did, tcx), None);
     tcx.arena.alloc(inner_optimized_mir(tcx, did))
 }
@@ -615,7 +612,7 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> {
         // Run the `mir_for_ctfe` query, which depends on `mir_drops_elaborated_and_const_checked`
         // which we are going to steal below. Thus we need to run `mir_for_ctfe` first, so it
         // computes and caches its result.
-        Some(hir::ConstContext::ConstFn) => tcx.ensure().mir_for_ctfe(did),
+        Some(hir::ConstContext::ConstFn) => tcx.ensure_with_value().mir_for_ctfe(did),
         None => {}
         Some(other) => panic!("do not use `optimized_mir` for constants: {:?}", other),
     }
@@ -626,8 +623,6 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> {
     debug!("body: {:#?}", body);
     run_optimization_passes(tcx, &mut body);
 
-    debug_assert!(!body.has_free_regions(), "Free regions in optimized MIR");
-
     body
 }
 
@@ -651,7 +646,5 @@ fn promoted_mir(
         run_analysis_to_runtime_passes(tcx, body);
     }
 
-    debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR");
-
     tcx.arena.alloc(promoted)
 }
diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
index f596cc180..c136642df 100644
--- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs
+++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
@@ -6,6 +6,7 @@ use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 
 pub struct LowerIntrinsics;
 
@@ -149,6 +150,35 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                             terminator.kind = TerminatorKind::Goto { target };
                         }
                     }
+                    sym::read_via_copy => {
+                        let [arg] = args.as_slice() else {
+                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
+                        };
+                        let derefed_place =
+                            if let Some(place) = arg.place() && let Some(local) = place.as_local() {
+                                tcx.mk_place_deref(local.into())
+                            } else {
+                                span_bug!(terminator.source_info.span, "Only passing a local is supported");
+                            };
+                        terminator.kind = match *target {
+                            None => {
+                                // No target means this read something uninhabited,
+                                // so it must be unreachable, and we don't need to
+                                // preserve the assignment either.
+                                TerminatorKind::Unreachable
+                            }
+                            Some(target) => {
+                                block.statements.push(Statement {
+                                    source_info: terminator.source_info,
+                                    kind: StatementKind::Assign(Box::new((
+                                        *destination,
+                                        Rvalue::Use(Operand::Copy(derefed_place)),
+                                    ))),
+                                });
+                                TerminatorKind::Goto { target }
+                            }
+                        }
+                    }
                     sym::discriminant_value => {
                         if let (Some(target), Some(arg)) = (*target, args[0].place()) {
                             let arg = tcx.mk_place_deref(arg);
@@ -162,6 +192,61 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                             terminator.kind = TerminatorKind::Goto { target };
                         }
                     }
+                    sym::option_payload_ptr => {
+                        if let (Some(target), Some(arg)) = (*target, args[0].place()) {
+                            let ty::RawPtr(ty::TypeAndMut { ty: dest_ty, .. }) =
+                                destination.ty(local_decls, tcx).ty.kind()
+                            else { bug!(); };
+
+                            block.statements.push(Statement {
+                                source_info: terminator.source_info,
+                                kind: StatementKind::Assign(Box::new((
+                                    *destination,
+                                    Rvalue::AddressOf(
+                                        Mutability::Not,
+                                        arg.project_deeper(
+                                            &[
+                                                PlaceElem::Deref,
+                                                PlaceElem::Downcast(
+                                                    Some(sym::Some),
+                                                    VariantIdx::from_u32(1),
+                                                ),
+                                                PlaceElem::Field(FieldIdx::from_u32(0), *dest_ty),
+                                            ],
+                                            tcx,
+                                        ),
+                                    ),
+                                ))),
+                            });
+                            terminator.kind = TerminatorKind::Goto { target };
+                        }
+                    }
+                    sym::transmute => {
+                        let dst_ty = destination.ty(local_decls, tcx).ty;
+                        let Ok([arg]) = <[_; 1]>::try_from(std::mem::take(args)) else {
+                            span_bug!(
+                                terminator.source_info.span,
+                                "Wrong number of arguments for transmute intrinsic",
+                            );
+                        };
+
+                        // Always emit the cast, even if we transmute to an uninhabited type,
+                        // because that lets CTFE and codegen generate better error messages
+                        // when such a transmute actually ends up reachable.
+                        block.statements.push(Statement {
+                            source_info: terminator.source_info,
+                            kind: StatementKind::Assign(Box::new((
+                                *destination,
+                                Rvalue::Cast(CastKind::Transmute, arg, dst_ty),
+                            ))),
+                        });
+
+                        if let Some(target) = *target {
+                            terminator.kind = TerminatorKind::Goto { target };
+                        } else {
+                            terminator.kind = TerminatorKind::Unreachable;
+                        }
+                    }
                     _ if intrinsic_name.as_str().starts_with("simd_shuffle") => {
                         validate_simd_shuffle(tcx, args, terminator.source_info.span);
                     }
diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs
index c6e7468aa..7dc5878e0 100644
--- a/compiler/rustc_mir_transform/src/lower_slice_len.rs
+++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs
@@ -3,7 +3,7 @@
 
 use crate::MirPass;
 use rustc_hir::def_id::DefId;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::IndexSlice;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, TyCtxt};
 
@@ -42,7 +42,7 @@ struct SliceLenPatchInformation<'tcx> {
 fn lower_slice_len_call<'tcx>(
     tcx: TyCtxt<'tcx>,
     block: &mut BasicBlockData<'tcx>,
-    local_decls: &IndexVec<Local, LocalDecl<'tcx>>,
+    local_decls: &IndexSlice<Local, LocalDecl<'tcx>>,
     slice_len_fn_item_def_id: DefId,
 ) {
     let mut patch_found: Option<SliceLenPatchInformation<'_>> = None;
@@ -54,7 +54,6 @@ fn lower_slice_len_call<'tcx>(
             args,
             destination,
             target: Some(bb),
-            cleanup: None,
             from_hir_call: true,
             ..
         } => {
diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs
index 4291e81c7..b6e73eaad 100644
--- a/compiler/rustc_mir_transform/src/nrvo.rs
+++ b/compiler/rustc_mir_transform/src/nrvo.rs
@@ -102,7 +102,7 @@ fn local_eligible_for_nrvo(body: &mut mir::Body<'_>) -> Option<Local> {
             mir::LocalKind::Arg => return None,
 
             mir::LocalKind::ReturnPointer => bug!("Return place was assigned to itself?"),
-            mir::LocalKind::Var | mir::LocalKind::Temp => {}
+            mir::LocalKind::Temp => {}
         }
 
         // If multiple different locals are copied to the return place. We can't pick a
diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
index e3a03aa08..4941c9edc 100644
--- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
+++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
@@ -33,6 +33,7 @@ impl RemoveNoopLandingPads {
                 StatementKind::FakeRead(..)
                 | StatementKind::StorageLive(_)
                 | StatementKind::StorageDead(_)
+                | StatementKind::PlaceMention(..)
                 | StatementKind::AscribeUserType(..)
                 | StatementKind::Coverage(..)
                 | StatementKind::ConstEvalCounter
@@ -71,11 +72,10 @@ impl RemoveNoopLandingPads {
             TerminatorKind::GeneratorDrop
             | TerminatorKind::Yield { .. }
             | TerminatorKind::Return
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Unreachable
             | TerminatorKind::Call { .. }
             | TerminatorKind::Assert { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::Drop { .. }
             | TerminatorKind::InlineAsm { .. } => false,
         }
@@ -103,11 +103,11 @@ impl RemoveNoopLandingPads {
         for bb in postorder {
             debug!("  processing {:?}", bb);
             if let Some(unwind) = body[bb].terminator_mut().unwind_mut() {
-                if let Some(unwind_bb) = *unwind {
+                if let UnwindAction::Cleanup(unwind_bb) = *unwind {
                     if nop_landing_pads.contains(unwind_bb) {
                         debug!("    removing noop landing pad");
                         landing_pads_removed += 1;
-                        *unwind = None;
+                        *unwind = UnwindAction::Continue;
                     }
                 }
             }
diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
index 78b6f714a..1f9e521d3 100644
--- a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
+++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
@@ -1,14 +1,15 @@
 use rustc_index::bit_set::ChunkedBitSet;
-use rustc_middle::mir::{Body, Field, Rvalue, Statement, StatementKind, TerminatorKind};
+use rustc_middle::mir::{Body, TerminatorKind};
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt, VariantDef};
 use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex};
 use rustc_mir_dataflow::{self, move_path_children_matching, Analysis, MoveDataParamEnv};
+use rustc_target::abi::FieldIdx;
 
 use crate::MirPass;
 
-/// Removes `Drop` and `DropAndReplace` terminators whose target is known to be uninitialized at
+/// Removes `Drop` terminators whose target is known to be uninitialized at
 /// that point.
 ///
 /// This is redundant with drop elaboration, but we need to do it prior to const-checking, and
@@ -37,8 +38,7 @@ impl<'tcx> MirPass<'tcx> for RemoveUninitDrops {
         let mut to_remove = vec![];
         for (bb, block) in body.basic_blocks.iter_enumerated() {
             let terminator = block.terminator();
-            let (TerminatorKind::Drop { place, .. } | TerminatorKind::DropAndReplace { place, .. })
-                = &terminator.kind
+            let TerminatorKind::Drop { place, .. } = &terminator.kind
             else { continue };
 
             maybe_inits.seek_before_primary_effect(body.terminator_loc(bb));
@@ -64,24 +64,12 @@ impl<'tcx> MirPass<'tcx> for RemoveUninitDrops {
         for bb in to_remove {
             let block = &mut body.basic_blocks_mut()[bb];
 
-            let (TerminatorKind::Drop { target, .. } | TerminatorKind::DropAndReplace { target, .. })
+            let TerminatorKind::Drop { target, .. }
                 = &block.terminator().kind
             else { unreachable!() };
 
             // Replace block terminator with `Goto`.
-            let target = *target;
-            let old_terminator_kind = std::mem::replace(
-                &mut block.terminator_mut().kind,
-                TerminatorKind::Goto { target },
-            );
-
-            // If this is a `DropAndReplace`, we need to emulate the assignment to the return place.
-            if let TerminatorKind::DropAndReplace { place, value, .. } = old_terminator_kind {
-                block.statements.push(Statement {
-                    source_info: block.terminator().source_info,
-                    kind: StatementKind::Assign(Box::new((place, Rvalue::Use(value)))),
-                });
-            }
+            block.terminator_mut().kind = TerminatorKind::Goto { target: *target };
         }
     }
 }
@@ -143,7 +131,7 @@ fn is_needs_drop_and_init<'tcx>(
                     .fields
                     .iter()
                     .enumerate()
-                    .map(|(f, field)| (Field::from_usize(f), field.ty(tcx, substs), mpi))
+                    .map(|(f, field)| (FieldIdx::from_usize(f), field.ty(tcx, substs), mpi))
                     .any(field_needs_drop_and_init)
             })
         }
@@ -151,7 +139,7 @@ fn is_needs_drop_and_init<'tcx>(
         ty::Tuple(fields) => fields
             .iter()
             .enumerate()
-            .map(|(f, f_ty)| (Field::from_usize(f), f_ty, mpi))
+            .map(|(f, f_ty)| (FieldIdx::from_usize(f), f_ty, mpi))
             .any(field_needs_drop_and_init),
 
         _ => true,
diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs
index 1becfddb2..1f37f03cf 100644
--- a/compiler/rustc_mir_transform/src/remove_zsts.rs
+++ b/compiler/rustc_mir_transform/src/remove_zsts.rs
@@ -1,7 +1,9 @@
-//! Removes assignments to ZST places.
+//! Removes operations on ZST places, and convert ZST operands to constants.
 
 use crate::MirPass;
-use rustc_middle::mir::{Body, StatementKind};
+use rustc_middle::mir::interpret::ConstValue;
+use rustc_middle::mir::visit::*;
+use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 
 pub struct RemoveZsts;
@@ -16,38 +18,24 @@ impl<'tcx> MirPass<'tcx> for RemoveZsts {
         if tcx.type_of(body.source.def_id()).subst_identity().is_generator() {
             return;
         }
-        let param_env = tcx.param_env(body.source.def_id());
-        let basic_blocks = body.basic_blocks.as_mut_preserves_cfg();
+        let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
         let local_decls = &body.local_decls;
-        for block in basic_blocks {
-            for statement in block.statements.iter_mut() {
-                if let StatementKind::Assign(box (place, _)) | StatementKind::Deinit(box place) =
-                    statement.kind
-                {
-                    let place_ty = place.ty(local_decls, tcx).ty;
-                    if !maybe_zst(place_ty) {
-                        continue;
-                    }
-                    let Ok(layout) = tcx.layout_of(param_env.and(place_ty)) else {
-                        continue;
-                    };
-                    if !layout.is_zst() {
-                        continue;
-                    }
-                    if tcx.consider_optimizing(|| {
-                        format!(
-                            "RemoveZsts - Place: {:?} SourceInfo: {:?}",
-                            place, statement.source_info
-                        )
-                    }) {
-                        statement.make_nop();
-                    }
-                }
-            }
+        let mut replacer = Replacer { tcx, param_env, local_decls };
+        for var_debug_info in &mut body.var_debug_info {
+            replacer.visit_var_debug_info(var_debug_info);
+        }
+        for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() {
+            replacer.visit_basic_block_data(bb, data);
         }
     }
 }
 
+struct Replacer<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    local_decls: &'a LocalDecls<'tcx>,
+}
+
 /// A cheap, approximate check to avoid unnecessary `layout_of` calls.
 fn maybe_zst(ty: Ty<'_>) -> bool {
     match ty.kind() {
@@ -63,3 +51,93 @@ fn maybe_zst(ty: Ty<'_>) -> bool {
         _ => false,
     }
 }
+
+impl<'tcx> Replacer<'_, 'tcx> {
+    fn known_to_be_zst(&self, ty: Ty<'tcx>) -> bool {
+        if !maybe_zst(ty) {
+            return false;
+        }
+        let Ok(layout) = self.tcx.layout_of(self.param_env.and(ty)) else {
+            return false;
+        };
+        layout.is_zst()
+    }
+
+    fn make_zst(&self, ty: Ty<'tcx>) -> Constant<'tcx> {
+        debug_assert!(self.known_to_be_zst(ty));
+        Constant {
+            span: rustc_span::DUMMY_SP,
+            user_ty: None,
+            literal: ConstantKind::Val(ConstValue::ZeroSized, ty),
+        }
+    }
+}
+
+impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    fn visit_var_debug_info(&mut self, var_debug_info: &mut VarDebugInfo<'tcx>) {
+        match var_debug_info.value {
+            VarDebugInfoContents::Const(_) => {}
+            VarDebugInfoContents::Place(place) => {
+                let place_ty = place.ty(self.local_decls, self.tcx).ty;
+                if self.known_to_be_zst(place_ty) {
+                    var_debug_info.value = VarDebugInfoContents::Const(self.make_zst(place_ty))
+                }
+            }
+            VarDebugInfoContents::Composite { ty, fragments: _ } => {
+                if self.known_to_be_zst(ty) {
+                    var_debug_info.value = VarDebugInfoContents::Const(self.make_zst(ty))
+                }
+            }
+        }
+    }
+
+    fn visit_operand(&mut self, operand: &mut Operand<'tcx>, loc: Location) {
+        if let Operand::Constant(_) = operand {
+            return;
+        }
+        let op_ty = operand.ty(self.local_decls, self.tcx);
+        if self.known_to_be_zst(op_ty)
+            && self.tcx.consider_optimizing(|| {
+                format!("RemoveZsts - Operand: {:?} Location: {:?}", operand, loc)
+            })
+        {
+            *operand = Operand::Constant(Box::new(self.make_zst(op_ty)))
+        }
+    }
+
+    fn visit_statement(&mut self, statement: &mut Statement<'tcx>, loc: Location) {
+        let place_for_ty = match statement.kind {
+            StatementKind::Assign(box (place, ref rvalue)) => {
+                rvalue.is_safe_to_remove().then_some(place)
+            }
+            StatementKind::Deinit(box place)
+            | StatementKind::SetDiscriminant { box place, variant_index: _ }
+            | StatementKind::AscribeUserType(box (place, _), _)
+            | StatementKind::Retag(_, box place)
+            | StatementKind::PlaceMention(box place)
+            | StatementKind::FakeRead(box (_, place)) => Some(place),
+            StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => {
+                Some(local.into())
+            }
+            StatementKind::Coverage(_)
+            | StatementKind::Intrinsic(_)
+            | StatementKind::Nop
+            | StatementKind::ConstEvalCounter => None,
+        };
+        if let Some(place_for_ty) = place_for_ty
+            && let ty = place_for_ty.ty(self.local_decls, self.tcx).ty
+            && self.known_to_be_zst(ty)
+            && self.tcx.consider_optimizing(|| {
+                format!("RemoveZsts - Place: {:?} SourceInfo: {:?}", place_for_ty, statement.source_info)
+            })
+        {
+            statement.make_nop();
+        } else {
+            self.super_statement(statement, loc);
+        }
+    }
+}
diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs
index a24d2d34d..ef367faf6 100644
--- a/compiler/rustc_mir_transform/src/separate_const_switch.rs
+++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs
@@ -108,12 +108,11 @@ pub fn separate_const_switch(body: &mut Body<'_>) -> usize {
                         // The following terminators are not allowed
                         TerminatorKind::Resume
                         | TerminatorKind::Drop { .. }
-                        | TerminatorKind::DropAndReplace { .. }
                         | TerminatorKind::Call { .. }
                         | TerminatorKind::Assert { .. }
                         | TerminatorKind::FalseUnwind { .. }
                         | TerminatorKind::Yield { .. }
-                        | TerminatorKind::Abort
+                        | TerminatorKind::Terminate
                         | TerminatorKind::Return
                         | TerminatorKind::Unreachable
                         | TerminatorKind::InlineAsm { .. }
@@ -165,12 +164,11 @@ pub fn separate_const_switch(body: &mut Body<'_>) -> usize {
             }
 
             TerminatorKind::Resume
-            | TerminatorKind::Abort
+            | TerminatorKind::Terminate
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::GeneratorDrop
             | TerminatorKind::Assert { .. }
-            | TerminatorKind::DropAndReplace { .. }
             | TerminatorKind::FalseUnwind { .. }
             | TerminatorKind::Drop { .. }
             | TerminatorKind::Call { .. }
@@ -247,6 +245,7 @@ fn is_likely_const<'tcx>(mut tracked_place: Place<'tcx>, block: &BasicBlockData<
             | StatementKind::StorageLive(_)
             | StatementKind::Retag(_, _)
             | StatementKind::AscribeUserType(_, _)
+            | StatementKind::PlaceMention(..)
             | StatementKind::Coverage(_)
             | StatementKind::StorageDead(_)
             | StatementKind::Intrinsic(_)
@@ -317,6 +316,7 @@ fn find_determining_place<'tcx>(
             | StatementKind::StorageDead(_)
             | StatementKind::Retag(_, _)
             | StatementKind::AscribeUserType(_, _)
+            | StatementKind::PlaceMention(..)
             | StatementKind::Coverage(_)
             | StatementKind::Intrinsic(_)
             | StatementKind::ConstEvalCounter
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index ebe63d6cb..2787fe2ce 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -5,7 +5,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::InternalSubsts;
 use rustc_middle::ty::{self, EarlyBinder, GeneratorSubsts, Ty, TyCtxt};
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT};
 
 use rustc_index::vec::{Idx, IndexVec};
 
@@ -76,7 +76,9 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'
 
             build_drop_shim(tcx, def_id, ty)
         }
+        ty::InstanceDef::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance),
         ty::InstanceDef::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty),
+        ty::InstanceDef::FnPtrAddrShim(def_id, ty) => build_fn_ptr_addr_shim(tcx, def_id, ty),
         ty::InstanceDef::Virtual(..) => {
             bug!("InstanceDef::Virtual ({:?}) is for direct calls only", instance)
         }
@@ -307,7 +309,7 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> {
 
     fn clear_drop_flag(&mut self, _location: Location, _path: Self::Path, _mode: DropFlagMode) {}
 
-    fn field_subpath(&self, _path: Self::Path, _field: Field) -> Option<Self::Path> {
+    fn field_subpath(&self, _path: Self::Path, _field: FieldIdx) -> Option<Self::Path> {
         None
     }
     fn deref_subpath(&self, _path: Self::Path) -> Option<Self::Path> {
@@ -321,6 +323,34 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> {
     }
 }
 
+fn build_thread_local_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'tcx> {
+    let def_id = instance.def_id();
+
+    let span = tcx.def_span(def_id);
+    let source_info = SourceInfo::outermost(span);
+
+    let mut blocks = IndexVec::with_capacity(1);
+    blocks.push(BasicBlockData {
+        statements: vec![Statement {
+            source_info,
+            kind: StatementKind::Assign(Box::new((
+                Place::return_place(),
+                Rvalue::ThreadLocalRef(def_id),
+            ))),
+        }],
+        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
+        is_cleanup: false,
+    });
+
+    new_body(
+        MirSource::from_instance(instance),
+        blocks,
+        IndexVec::from_raw(vec![LocalDecl::new(tcx.thread_local_ptr_ty(def_id), span)]),
+        0,
+        span,
+    )
+}
+
 /// Builds a `Clone::clone` shim for `self_ty`. Here, `def_id` is `Clone::clone`.
 fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
     debug!("build_clone_shim(def_id={:?})", def_id);
@@ -469,7 +499,7 @@ impl<'tcx> CloneShimBuilder<'tcx> {
                 args: vec![Operand::Move(ref_loc)],
                 destination: dest,
                 target: Some(next),
-                cleanup: Some(cleanup),
+                unwind: UnwindAction::Cleanup(cleanup),
                 from_hir_call: true,
                 fn_span: self.span,
             },
@@ -500,7 +530,7 @@ impl<'tcx> CloneShimBuilder<'tcx> {
             // created by block 2*i. We store this block in `unwind` so that the next clone block
             // will unwind to it if cloning fails.
 
-            let field = Field::new(i);
+            let field = FieldIdx::new(i);
             let src_field = self.tcx.mk_place_field(src, field, ity);
 
             let dest_field = self.tcx.mk_place_field(dest, field, ity);
@@ -510,7 +540,11 @@ impl<'tcx> CloneShimBuilder<'tcx> {
             self.make_clone_call(dest_field, src_field, ity, next_block, unwind);
             self.block(
                 vec![],
-                TerminatorKind::Drop { place: dest_field, target: unwind, unwind: None },
+                TerminatorKind::Drop {
+                    place: dest_field,
+                    target: unwind,
+                    unwind: UnwindAction::Terminate,
+                },
                 true,
             );
             unwind = next_unwind;
@@ -723,7 +757,7 @@ fn build_call_shim<'tcx>(
     if let Some(untuple_args) = untuple_args {
         let tuple_arg = Local::new(1 + (sig.inputs().len() - 1));
         args.extend(untuple_args.iter().enumerate().map(|(i, ity)| {
-            Operand::Move(tcx.mk_place_field(Place::from(tuple_arg), Field::new(i), *ity))
+            Operand::Move(tcx.mk_place_field(Place::from(tuple_arg), FieldIdx::new(i), *ity))
         }));
     }
 
@@ -746,10 +780,10 @@ fn build_call_shim<'tcx>(
             args,
             destination: Place::return_place(),
             target: Some(BasicBlock::new(1)),
-            cleanup: if let Some(Adjustment::RefMut) = rcvr_adjustment {
-                Some(BasicBlock::new(3))
+            unwind: if let Some(Adjustment::RefMut) = rcvr_adjustment {
+                UnwindAction::Cleanup(BasicBlock::new(3))
             } else {
-                None
+                UnwindAction::Continue
             },
             from_hir_call: true,
             fn_span: span,
@@ -762,7 +796,11 @@ fn build_call_shim<'tcx>(
         block(
             &mut blocks,
             vec![],
-            TerminatorKind::Drop { place: rcvr_place(), target: BasicBlock::new(2), unwind: None },
+            TerminatorKind::Drop {
+                place: rcvr_place(),
+                target: BasicBlock::new(2),
+                unwind: UnwindAction::Continue,
+            },
             false,
         );
     }
@@ -773,7 +811,11 @@ fn build_call_shim<'tcx>(
         block(
             &mut blocks,
             vec![],
-            TerminatorKind::Drop { place: rcvr_place(), target: BasicBlock::new(4), unwind: None },
+            TerminatorKind::Drop {
+                place: rcvr_place(),
+                target: BasicBlock::new(4),
+                unwind: UnwindAction::Terminate,
+            },
             true,
         );
 
@@ -816,11 +858,8 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
 
     let source_info = SourceInfo::outermost(span);
 
-    let variant_index = if adt_def.is_enum() {
-        adt_def.variant_index_with_ctor_id(ctor_id)
-    } else {
-        VariantIdx::new(0)
-    };
+    let variant_index =
+        if adt_def.is_enum() { adt_def.variant_index_with_ctor_id(ctor_id) } else { FIRST_VARIANT };
 
     // Generate the following MIR:
     //
@@ -864,3 +903,39 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
 
     body
 }
+
+/// ```ignore (pseudo-impl)
+/// impl FnPtr for fn(u32) {
+///     fn addr(self) -> usize {
+///         self as usize
+///     }
+/// }
+/// ```
+fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
+    assert!(matches!(self_ty.kind(), ty::FnPtr(..)), "expected fn ptr, found {self_ty}");
+    let span = tcx.def_span(def_id);
+    let Some(sig) = tcx.fn_sig(def_id).subst(tcx, &[self_ty.into()]).no_bound_vars() else {
+        span_bug!(span, "FnPtr::addr with bound vars for `{self_ty}`");
+    };
+    let locals = local_decls_for_sig(&sig, span);
+
+    let source_info = SourceInfo::outermost(span);
+    // FIXME: use `expose_addr` once we figure out whether function pointers have meaningful provenance.
+    let rvalue = Rvalue::Cast(
+        CastKind::FnPtrToPtr,
+        Operand::Move(Place::from(Local::new(1))),
+        tcx.mk_imm_ptr(tcx.types.unit),
+    );
+    let stmt = Statement {
+        source_info,
+        kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
+    };
+    let statements = vec![stmt];
+    let start_block = BasicBlockData {
+        statements,
+        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
+        is_cleanup: false,
+    };
+    let source = MirSource::from_instance(ty::InstanceDef::FnPtrAddrShim(def_id, self_ty));
+    new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span)
+}
diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs
index 9ef55c558..c79e1cf08 100644
--- a/compiler/rustc_mir_transform/src/simplify.rs
+++ b/compiler/rustc_mir_transform/src/simplify.rs
@@ -28,8 +28,8 @@
 //! return.
 
 use crate::MirPass;
-use rustc_data_structures::fx::FxHashSet;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
+use rustc_index::vec::{Idx, IndexSlice, IndexVec};
 use rustc_middle::mir::coverage::*;
 use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::*;
@@ -48,6 +48,7 @@ impl SimplifyCfg {
 
 pub fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     CfgSimplifier::new(body).simplify();
+    remove_duplicate_unreachable_blocks(tcx, body);
     remove_dead_blocks(tcx, body);
 
     // FIXME: Should probably be moved into some kind of pass manager
@@ -66,7 +67,7 @@ impl<'tcx> MirPass<'tcx> for SimplifyCfg {
 }
 
 pub struct CfgSimplifier<'a, 'tcx> {
-    basic_blocks: &'a mut IndexVec<BasicBlock, BasicBlockData<'tcx>>,
+    basic_blocks: &'a mut IndexSlice<BasicBlock, BasicBlockData<'tcx>>,
     pred_count: IndexVec<BasicBlock, u32>,
 }
 
@@ -259,6 +260,49 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
     }
 }
 
+pub fn remove_duplicate_unreachable_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    struct OptApplier<'tcx> {
+        tcx: TyCtxt<'tcx>,
+        duplicates: FxIndexSet<BasicBlock>,
+    }
+
+    impl<'tcx> MutVisitor<'tcx> for OptApplier<'tcx> {
+        fn tcx(&self) -> TyCtxt<'tcx> {
+            self.tcx
+        }
+
+        fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) {
+            for target in terminator.successors_mut() {
+                // We don't have to check whether `target` is a cleanup block, because have
+                // entirely excluded cleanup blocks in building the set of duplicates.
+                if self.duplicates.contains(target) {
+                    *target = self.duplicates[0];
+                }
+            }
+
+            self.super_terminator(terminator, location);
+        }
+    }
+
+    let unreachable_blocks = body
+        .basic_blocks
+        .iter_enumerated()
+        .filter(|(_, bb)| {
+            // CfgSimplifier::simplify leaves behind some unreachable basic blocks without a
+            // terminator. Those blocks will be deleted by remove_dead_blocks, but we run just
+            // before then so we need to handle missing terminators.
+            // We also need to prevent confusing cleanup and non-cleanup blocks. In practice we
+            // don't emit empty unreachable cleanup blocks, so this simple check suffices.
+            bb.terminator.is_some() && bb.is_empty_unreachable() && !bb.is_cleanup
+        })
+        .map(|(block, _)| block)
+        .collect::<FxIndexSet<_>>();
+
+    if unreachable_blocks.len() > 1 {
+        OptApplier { tcx, duplicates: unreachable_blocks }.visit_body(body);
+    }
+}
+
 pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let reachable = traversal::reachable_as_bitset(body);
     let num_blocks = body.basic_blocks.len();
@@ -325,8 +369,8 @@ pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 /// instances in a single body, so the strategy described above is applied to
 /// coverage counters from each instance individually.
 fn save_unreachable_coverage(
-    basic_blocks: &mut IndexVec<BasicBlock, BasicBlockData<'_>>,
-    source_scopes: &IndexVec<SourceScope, SourceScopeData<'_>>,
+    basic_blocks: &mut IndexSlice<BasicBlock, BasicBlockData<'_>>,
+    source_scopes: &IndexSlice<SourceScope, SourceScopeData<'_>>,
     first_dead_block: usize,
 ) {
     // Identify instances that still have some live coverage counters left.
@@ -445,7 +489,7 @@ fn make_local_map<V>(
     local_decls: &mut IndexVec<Local, V>,
     used_locals: &UsedLocals,
 ) -> IndexVec<Local, Option<Local>> {
-    let mut map: IndexVec<Local, Option<Local>> = IndexVec::from_elem(None, &*local_decls);
+    let mut map: IndexVec<Local, Option<Local>> = IndexVec::from_elem(None, local_decls);
     let mut used = Local::new(0);
 
     for alive_index in local_decls.indices() {
@@ -525,6 +569,7 @@ impl<'tcx> Visitor<'tcx> for UsedLocals {
             | StatementKind::Retag(..)
             | StatementKind::Coverage(..)
             | StatementKind::FakeRead(..)
+            | StatementKind::PlaceMention(..)
             | StatementKind::AscribeUserType(..) => {
                 self.super_statement(statement, location);
             }
diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs
index 13168e9a2..c798bd053 100644
--- a/compiler/rustc_mir_transform/src/sroa.rs
+++ b/compiler/rustc_mir_transform/src/sroa.rs
@@ -4,8 +4,9 @@ use rustc_index::vec::IndexVec;
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
-use rustc_middle::ty::{Ty, TyCtxt};
+use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_mir_dataflow::value_analysis::{excluded_locals, iter_fields};
+use rustc_target::abi::FieldIdx;
 
 pub struct ScalarReplacementOfAggregates;
 
@@ -18,11 +19,12 @@ impl<'tcx> MirPass<'tcx> for ScalarReplacementOfAggregates {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         debug!(def_id = ?body.source.def_id());
         let mut excluded = excluded_locals(body);
+        let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
         loop {
             debug!(?excluded);
             let escaping = escaping_locals(&excluded, body);
             debug!(?escaping);
-            let replacements = compute_flattening(tcx, body, escaping);
+            let replacements = compute_flattening(tcx, param_env, body, escaping);
             debug!(?replacements);
             let all_dead_locals = replace_flattened_locals(tcx, body, replacements);
             if !all_dead_locals.is_empty() {
@@ -114,7 +116,7 @@ fn escaping_locals(excluded: &BitSet<Local>, body: &Body<'_>) -> BitSet<Local> {
 struct ReplacementMap<'tcx> {
     /// Pre-computed list of all "new" locals for each "old" local. This is used to expand storage
     /// and deinit statement and debuginfo.
-    fragments: IndexVec<Local, Option<IndexVec<Field, Option<(Ty<'tcx>, Local)>>>>,
+    fragments: IndexVec<Local, Option<IndexVec<FieldIdx, Option<(Ty<'tcx>, Local)>>>>,
 }
 
 impl<'tcx> ReplacementMap<'tcx> {
@@ -128,7 +130,7 @@ impl<'tcx> ReplacementMap<'tcx> {
     fn place_fragments(
         &self,
         place: Place<'tcx>,
-    ) -> Option<impl Iterator<Item = (Field, Ty<'tcx>, Local)> + '_> {
+    ) -> Option<impl Iterator<Item = (FieldIdx, Ty<'tcx>, Local)> + '_> {
         let local = place.as_local()?;
         let fields = self.fragments[local].as_ref()?;
         Some(fields.iter_enumerated().filter_map(|(field, &opt_ty_local)| {
@@ -144,6 +146,7 @@ impl<'tcx> ReplacementMap<'tcx> {
 /// The replacement will be done later in `ReplacementVisitor`.
 fn compute_flattening<'tcx>(
     tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
     body: &mut Body<'tcx>,
     escaping: BitSet<Local>,
 ) -> ReplacementMap<'tcx> {
@@ -155,7 +158,7 @@ fn compute_flattening<'tcx>(
         }
         let decl = body.local_decls[local].clone();
         let ty = decl.ty;
-        iter_fields(ty, tcx, |variant, field, field_ty| {
+        iter_fields(ty, tcx, param_env, |variant, field, field_ty| {
             if variant.is_some() {
                 // Downcasts are currently not supported.
                 return;
diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs
index c1e7f62de..be026402d 100644
--- a/compiler/rustc_mir_transform/src/ssa.rs
+++ b/compiler/rustc_mir_transform/src/ssa.rs
@@ -1,7 +1,7 @@
 use either::Either;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_index::bit_set::BitSet;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::middle::resolve_bound_vars::Set1;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
@@ -53,7 +53,7 @@ impl SsaLocals {
         body: &Body<'tcx>,
         borrowed_locals: &BitSet<Local>,
     ) -> SsaLocals {
-        let assignment_order = Vec::new();
+        let assignment_order = Vec::with_capacity(body.local_decls.len());
 
         let assignments = IndexVec::from_elem(Set1::Empty, &body.local_decls);
         let dominators =
@@ -135,7 +135,7 @@ impl SsaLocals {
     ///   _d => _a // transitively through _c
     ///
     /// Exception: we do not see through the return place, as it cannot be substituted.
-    pub fn copy_classes(&self) -> &IndexVec<Local, Local> {
+    pub fn copy_classes(&self) -> &IndexSlice<Local, Local> {
         &self.copy_classes
     }
 
@@ -179,12 +179,34 @@ struct SsaVisitor {
     assignment_order: Vec<Local>,
 }
 
+impl SsaVisitor {
+    fn check_assignment_dominates(&mut self, local: Local, loc: Location) {
+        let set = &mut self.assignments[local];
+        let assign_dominates = match *set {
+            Set1::Empty | Set1::Many => false,
+            Set1::One(LocationExtended::Arg) => true,
+            Set1::One(LocationExtended::Plain(assign)) => {
+                assign.successor_within_block().dominates(loc, &self.dominators)
+            }
+        };
+        // We are visiting a use that is not dominated by an assignment.
+        // Either there is a cycle involved, or we are reading for uninitialized local.
+        // Bail out.
+        if !assign_dominates {
+            *set = Set1::Many;
+        }
+    }
+}
+
 impl<'tcx> Visitor<'tcx> for SsaVisitor {
     fn visit_local(&mut self, local: Local, ctxt: PlaceContext, loc: Location) {
         match ctxt {
             PlaceContext::MutatingUse(MutatingUseContext::Store) => {
                 self.assignments[local].insert(LocationExtended::Plain(loc));
-                self.assignment_order.push(local);
+                if let Set1::One(_) = self.assignments[local] {
+                    // Only record if SSA-like, to avoid growing the vector needlessly.
+                    self.assignment_order.push(local);
+                }
             }
             // Anything can happen with raw pointers, so remove them.
             PlaceContext::NonMutatingUse(NonMutatingUseContext::AddressOf)
@@ -192,24 +214,26 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor {
             // Immutable borrows are taken into account in `SsaLocals::new` by
             // removing non-freeze locals.
             PlaceContext::NonMutatingUse(_) => {
-                let set = &mut self.assignments[local];
-                let assign_dominates = match *set {
-                    Set1::Empty | Set1::Many => false,
-                    Set1::One(LocationExtended::Arg) => true,
-                    Set1::One(LocationExtended::Plain(assign)) => {
-                        assign.successor_within_block().dominates(loc, &self.dominators)
-                    }
-                };
-                // We are visiting a use that is not dominated by an assignment.
-                // Either there is a cycle involved, or we are reading for uninitialized local.
-                // Bail out.
-                if !assign_dominates {
-                    *set = Set1::Many;
-                }
+                self.check_assignment_dominates(local, loc);
             }
             PlaceContext::NonUse(_) => {}
         }
     }
+
+    fn visit_place(&mut self, place: &Place<'tcx>, ctxt: PlaceContext, loc: Location) {
+        if place.projection.first() == Some(&PlaceElem::Deref) {
+            // Do not do anything for storage statements and debuginfo.
+            if ctxt.is_use() {
+                // A use through a `deref` only reads from the local, and cannot write to it.
+                let new_ctxt = PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection);
+
+                self.visit_projection(place.as_ref(), new_ctxt, loc);
+                self.check_assignment_dominates(place.local, loc);
+            }
+            return;
+        }
+        self.super_place(place, ctxt, loc);
+    }
 }
 
 #[instrument(level = "trace", skip(ssa, body))]
diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs
index d4b1cfe43..bd1724bf8 100644
--- a/compiler/rustc_mir_transform/src/unreachable_prop.rs
+++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs
@@ -99,7 +99,7 @@ where
                 //
                 // This generates a `switchInt() -> [0: 0, 1: 1, otherwise: unreachable]`, which allows us or LLVM to
                 // turn it into just `x` later. Without the unreachable, such a transformation would be illegal.
-                // If the otherwise branch is unreachable, we can delete all other unreacahble targets, as they will
+                // If the otherwise branch is unreachable, we can delete all other unreachable targets, as they will
                 // still point to the unreachable and therefore not lose reachability information.
                 let reachable_iter = targets.iter().filter(|(_, bb)| !is_unreachable(*bb));
 
diff --git a/compiler/rustc_monomorphize/locales/en-US.ftl b/compiler/rustc_monomorphize/messages.ftl
index 6cea6a603..6cea6a603 100644
--- a/compiler/rustc_monomorphize/locales/en-US.ftl
+++ b/compiler/rustc_monomorphize/messages.ftl
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index 45e659eab..7bcff7e07 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -174,7 +174,7 @@
 //! regardless of whether it is actually needed or not.
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_data_structures::sync::{par_for_each_in, MTLock, MTRef};
+use rustc_data_structures::sync::{par_for_each_in, MTLock, MTLockRef};
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, DefIdMap, LocalDefId};
@@ -190,7 +190,8 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::query::TyCtxtAt;
 use rustc_middle::ty::subst::{GenericArgKind, InternalSubsts};
 use rustc_middle::ty::{
-    self, GenericParamDefKind, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, VtblEntry,
+    self, GenericParamDefKind, Instance, InstanceDef, Ty, TyCtxt, TypeFoldable, TypeVisitableExt,
+    VtblEntry,
 };
 use rustc_middle::{middle::codegen_fn_attrs::CodegenFnAttrFlags, mir::visit::TyContext};
 use rustc_session::config::EntryFnType;
@@ -340,8 +341,8 @@ pub fn collect_crate_mono_items(
     let recursion_limit = tcx.recursion_limit();
 
     {
-        let visited: MTRef<'_, _> = &mut visited;
-        let inlining_map: MTRef<'_, _> = &mut inlining_map;
+        let visited: MTLockRef<'_, _> = &mut visited;
+        let inlining_map: MTLockRef<'_, _> = &mut inlining_map;
 
         tcx.sess.time("monomorphization_collector_graph_walk", || {
             par_for_each_in(roots, |root| {
@@ -406,10 +407,10 @@ fn collect_roots(tcx: TyCtxt<'_>, mode: MonoItemCollectionMode) -> Vec<MonoItem<
 fn collect_items_rec<'tcx>(
     tcx: TyCtxt<'tcx>,
     starting_point: Spanned<MonoItem<'tcx>>,
-    visited: MTRef<'_, MTLock<FxHashSet<MonoItem<'tcx>>>>,
+    visited: MTLockRef<'_, FxHashSet<MonoItem<'tcx>>>,
     recursion_depths: &mut DefIdMap<usize>,
     recursion_limit: Limit,
-    inlining_map: MTRef<'_, MTLock<InliningMap<'tcx>>>,
+    inlining_map: MTLockRef<'_, InliningMap<'tcx>>,
 ) {
     if !visited.lock_mut().insert(starting_point.node) {
         // We've been here already, no need to search again.
@@ -462,6 +463,16 @@ fn collect_items_rec<'tcx>(
                     collect_miri(tcx, id, &mut neighbors);
                 }
             }
+
+            if tcx.needs_thread_local_shim(def_id) {
+                neighbors.push(respan(
+                    starting_point.span,
+                    MonoItem::Fn(Instance {
+                        def: InstanceDef::ThreadLocalShim(def_id),
+                        substs: InternalSubsts::empty(),
+                    }),
+                ));
+            }
         }
         MonoItem::Fn(instance) => {
             // Sanity check whether this ended up being collected accidentally
@@ -640,8 +651,8 @@ fn check_type_length_limit<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
         let (shrunk, written_to_path) = shrunk_instance_name(tcx, &instance);
         let span = tcx.def_span(instance.def_id());
         let mut path = PathBuf::new();
-        let was_written = if written_to_path.is_some() {
-            path = written_to_path.unwrap();
+        let was_written = if let Some(path2) = written_to_path {
+            path = path2;
             Some(())
         } else {
             None
@@ -808,8 +819,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
                 let callee_ty = self.monomorphize(callee_ty);
                 visit_fn_use(self.tcx, callee_ty, true, source, &mut self.output)
             }
-            mir::TerminatorKind::Drop { ref place, .. }
-            | mir::TerminatorKind::DropAndReplace { ref place, .. } => {
+            mir::TerminatorKind::Drop { ref place, .. } => {
                 let ty = place.ty(self.body, self.tcx).ty;
                 let ty = self.monomorphize(ty);
                 visit_drop_use(self.tcx, ty, true, source, self.output);
@@ -842,7 +852,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
                     self.output.push(create_fn_mono_item(tcx, instance, source));
                 }
             }
-            mir::TerminatorKind::Abort { .. } => {
+            mir::TerminatorKind::Terminate { .. } => {
                 let instance = Instance::mono(
                     tcx,
                     tcx.require_lang_item(LangItem::PanicCannotUnwind, Some(source)),
@@ -862,6 +872,16 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
             | mir::TerminatorKind::FalseUnwind { .. } => bug!(),
         }
 
+        if let Some(mir::UnwindAction::Terminate) = terminator.unwind() {
+            let instance = Instance::mono(
+                tcx,
+                tcx.require_lang_item(LangItem::PanicCannotUnwind, Some(source)),
+            );
+            if should_codegen_locally(tcx, &instance) {
+                self.output.push(create_fn_mono_item(tcx, instance, source));
+            }
+        }
+
         self.super_terminator(terminator, location);
     }
 
@@ -963,6 +983,9 @@ fn visit_instance_use<'tcx>(
                 bug!("{:?} being reified", instance);
             }
         }
+        ty::InstanceDef::ThreadLocalShim(..) => {
+            bug!("{:?} being reified", instance);
+        }
         ty::InstanceDef::DropGlue(_, None) => {
             // Don't need to emit noop drop glue if we are calling directly.
             if !is_direct_call {
@@ -975,7 +998,8 @@ fn visit_instance_use<'tcx>(
         | ty::InstanceDef::ClosureOnceShim { .. }
         | ty::InstanceDef::Item(..)
         | ty::InstanceDef::FnPtrShim(..)
-        | ty::InstanceDef::CloneShim(..) => {
+        | ty::InstanceDef::CloneShim(..)
+        | ty::InstanceDef::FnPtrAddrShim(..) => {
             output.push(create_fn_mono_item(tcx, instance, source));
         }
     }
@@ -1105,7 +1129,8 @@ fn find_vtable_types_for_unsizing<'tcx>(
             let target_fields = &target_adt_def.non_enum_variant().fields;
 
             assert!(
-                coerce_index < source_fields.len() && source_fields.len() == target_fields.len()
+                coerce_index.index() < source_fields.len()
+                    && source_fields.len() == target_fields.len()
             );
 
             find_vtable_types_for_unsizing(
@@ -1210,11 +1235,9 @@ impl<'v> RootCollector<'_, 'v> {
                 self.output.push(dummy_spanned(MonoItem::GlobalAsm(id)));
             }
             DefKind::Static(..) => {
-                debug!(
-                    "RootCollector: ItemKind::Static({})",
-                    self.tcx.def_path_str(id.owner_id.to_def_id())
-                );
-                self.output.push(dummy_spanned(MonoItem::Static(id.owner_id.to_def_id())));
+                let def_id = id.owner_id.to_def_id();
+                debug!("RootCollector: ItemKind::Static({})", self.tcx.def_path_str(def_id));
+                self.output.push(dummy_spanned(MonoItem::Static(def_id)));
             }
             DefKind::Const => {
                 // const items only generate mono items if they are
@@ -1331,7 +1354,35 @@ fn create_mono_items_for_default_impls<'tcx>(
         return;
     };
 
-    let trait_ref = trait_ref.subst_identity();
+    // Lifetimes never affect trait selection, so we are allowed to eagerly
+    // instantiate an instance of an impl method if the impl (and method,
+    // which we check below) is only parameterized over lifetime. In that case,
+    // we use the ReErased, which has no lifetime information associated with
+    // it, to validate whether or not the impl is legal to instantiate at all.
+    let only_region_params = |param: &ty::GenericParamDef, _: &_| match param.kind {
+        GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
+        GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
+            unreachable!(
+                "`own_requires_monomorphization` check means that \
+                we should have no type/const params"
+            )
+        }
+    };
+    let impl_substs = InternalSubsts::for_item(tcx, item.owner_id.to_def_id(), only_region_params);
+    let trait_ref = trait_ref.subst(tcx, impl_substs);
+
+    // Unlike 'lazy' monomorphization that begins by collecting items transitively
+    // called by `main` or other global items, when eagerly monomorphizing impl
+    // items, we never actually check that the predicates of this impl are satisfied
+    // in a empty reveal-all param env (i.e. with no assumptions).
+    //
+    // Even though this impl has no type or const substitutions, because we don't
+    // consider higher-ranked predicates such as `for<'a> &'a mut [u8]: Copy` to
+    // be trivially false. We must now check that the impl has no impossible-to-satisfy
+    // predicates.
+    if tcx.subst_and_check_impossible_predicates((item.owner_id.to_def_id(), impl_substs)) {
+        return;
+    }
 
     let param_env = ty::ParamEnv::reveal_all();
     let trait_ref = tcx.normalize_erasing_regions(param_env, trait_ref);
@@ -1345,12 +1396,9 @@ fn create_mono_items_for_default_impls<'tcx>(
             continue;
         }
 
-        let substs = InternalSubsts::for_item(tcx, method.def_id, |param, _| match param.kind {
-            GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
-            GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
-                trait_ref.substs[param.index as usize]
-            }
-        });
+        // As mentioned above, the method is legal to eagerly instantiate if it
+        // only has lifetime substitutions. This is validated by
+        let substs = trait_ref.substs.extend_to(tcx, method.def_id, only_region_params);
         let instance = ty::Instance::expect_resolve(tcx, param_env, method.def_id, substs);
 
         let mono_item = create_fn_mono_item(tcx, instance, DUMMY_SP);
diff --git a/compiler/rustc_monomorphize/src/lib.rs b/compiler/rustc_monomorphize/src/lib.rs
index f6b791f29..5000fb719 100644
--- a/compiler/rustc_monomorphize/src/lib.rs
+++ b/compiler/rustc_monomorphize/src/lib.rs
@@ -23,7 +23,7 @@ mod partitioning;
 mod polymorphize;
 mod util;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 fn custom_coerce_unsize_info<'tcx>(
     tcx: TyCtxtAt<'tcx>,
diff --git a/compiler/rustc_monomorphize/src/partitioning/default.rs b/compiler/rustc_monomorphize/src/partitioning/default.rs
index 2c56edd89..482b78d42 100644
--- a/compiler/rustc_monomorphize/src/partitioning/default.rs
+++ b/compiler/rustc_monomorphize/src/partitioning/default.rs
@@ -9,7 +9,7 @@ use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel
 use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, Linkage, Visibility};
 use rustc_middle::mir::mono::{InstantiationMode, MonoItem};
 use rustc_middle::ty::print::characteristic_def_id_of_type;
-use rustc_middle::ty::{self, visit::TypeVisitableExt, DefIdTree, InstanceDef, TyCtxt};
+use rustc_middle::ty::{self, visit::TypeVisitableExt, InstanceDef, TyCtxt};
 use rustc_span::symbol::Symbol;
 
 use super::PartitioningCx;
@@ -89,10 +89,7 @@ impl<'tcx> Partitioner<'tcx> for DefaultPartitioning {
         }
 
         PreInliningPartitioning {
-            codegen_units: codegen_units
-                .into_iter()
-                .map(|(_, codegen_unit)| codegen_unit)
-                .collect(),
+            codegen_units: codegen_units.into_values().map(|codegen_unit| codegen_unit).collect(),
             roots,
             internalization_candidates,
         }
@@ -278,7 +275,9 @@ fn characteristic_def_id_of_mono_item<'tcx>(
                 | ty::InstanceDef::Intrinsic(..)
                 | ty::InstanceDef::DropGlue(..)
                 | ty::InstanceDef::Virtual(..)
-                | ty::InstanceDef::CloneShim(..) => return None,
+                | ty::InstanceDef::CloneShim(..)
+                | ty::InstanceDef::ThreadLocalShim(..)
+                | ty::InstanceDef::FnPtrAddrShim(..) => return None,
             };
 
             // If this is a method, we want to put it into the same module as
@@ -391,6 +390,19 @@ fn mono_item_linkage_and_visibility<'tcx>(
 
 type CguNameCache = FxHashMap<(DefId, bool), Symbol>;
 
+fn static_visibility<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    can_be_internalized: &mut bool,
+    def_id: DefId,
+) -> Visibility {
+    if tcx.is_reachable_non_generic(def_id) {
+        *can_be_internalized = false;
+        default_visibility(tcx, def_id, false)
+    } else {
+        Visibility::Hidden
+    }
+}
+
 fn mono_item_visibility<'tcx>(
     tcx: TyCtxt<'tcx>,
     mono_item: &MonoItem<'tcx>,
@@ -402,21 +414,9 @@ fn mono_item_visibility<'tcx>(
         MonoItem::Fn(instance) => instance,
 
         // Misc handling for generics and such, but otherwise:
-        MonoItem::Static(def_id) => {
-            return if tcx.is_reachable_non_generic(*def_id) {
-                *can_be_internalized = false;
-                default_visibility(tcx, *def_id, false)
-            } else {
-                Visibility::Hidden
-            };
-        }
+        MonoItem::Static(def_id) => return static_visibility(tcx, can_be_internalized, *def_id),
         MonoItem::GlobalAsm(item_id) => {
-            return if tcx.is_reachable_non_generic(item_id.owner_id) {
-                *can_be_internalized = false;
-                default_visibility(tcx, item_id.owner_id.to_def_id(), false)
-            } else {
-                Visibility::Hidden
-            };
+            return static_visibility(tcx, can_be_internalized, item_id.owner_id.to_def_id());
         }
     };
 
@@ -424,6 +424,11 @@ fn mono_item_visibility<'tcx>(
         InstanceDef::Item(def) => def.did,
         InstanceDef::DropGlue(def_id, Some(_)) => def_id,
 
+        // We match the visiblity of statics here
+        InstanceDef::ThreadLocalShim(def_id) => {
+            return static_visibility(tcx, can_be_internalized, def_id);
+        }
+
         // These are all compiler glue and such, never exported, always hidden.
         InstanceDef::VTableShim(..)
         | InstanceDef::ReifyShim(..)
@@ -432,7 +437,8 @@ fn mono_item_visibility<'tcx>(
         | InstanceDef::Intrinsic(..)
         | InstanceDef::ClosureOnceShim { .. }
         | InstanceDef::DropGlue(..)
-        | InstanceDef::CloneShim(..) => return Visibility::Hidden,
+        | InstanceDef::CloneShim(..)
+        | InstanceDef::FnPtrAddrShim(..) => return Visibility::Hidden,
     };
 
     // The `start_fn` lang item is actually a monomorphized instance of a
diff --git a/compiler/rustc_monomorphize/src/partitioning/merging.rs b/compiler/rustc_monomorphize/src/partitioning/merging.rs
index 02bb8dea0..5c524a184 100644
--- a/compiler/rustc_monomorphize/src/partitioning/merging.rs
+++ b/compiler/rustc_monomorphize/src/partitioning/merging.rs
@@ -24,7 +24,7 @@ pub fn merge_codegen_units<'tcx>(
     // smallest into each other) we're sure to start off with a deterministic
     // order (sorted by name). This'll mean that if two cgus have the same size
     // the stable sort below will keep everything nice and deterministic.
-    codegen_units.sort_by(|a, b| a.name().as_str().partial_cmp(b.name().as_str()).unwrap());
+    codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str()));
 
     // This map keeps track of what got merged into what.
     let mut cgu_contents: FxHashMap<Symbol, Vec<Symbol>> =
diff --git a/compiler/rustc_monomorphize/src/partitioning/mod.rs b/compiler/rustc_monomorphize/src/partitioning/mod.rs
index 524c51d88..18aa0742c 100644
--- a/compiler/rustc_monomorphize/src/partitioning/mod.rs
+++ b/compiler/rustc_monomorphize/src/partitioning/mod.rs
@@ -252,7 +252,7 @@ pub fn partition<'tcx>(
         internalization_candidates: _,
     } = post_inlining;
 
-    result.sort_by(|a, b| a.name().as_str().partial_cmp(b.name().as_str()).unwrap());
+    result.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str()));
 
     result
 }
@@ -474,7 +474,7 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[Co
     (tcx.arena.alloc(mono_items), codegen_units)
 }
 
-/// Outputs stats about instantation counts and estimated size, per `MonoItem`'s
+/// Outputs stats about instantiation counts and estimated size, per `MonoItem`'s
 /// def, to a file in the given output directory.
 fn dump_mono_items_stats<'tcx>(
     tcx: TyCtxt<'tcx>,
diff --git a/compiler/rustc_monomorphize/src/polymorphize.rs b/compiler/rustc_monomorphize/src/polymorphize.rs
index b7c3dbcc0..63263a642 100644
--- a/compiler/rustc_monomorphize/src/polymorphize.rs
+++ b/compiler/rustc_monomorphize/src/polymorphize.rs
@@ -36,6 +36,8 @@ fn unused_generic_params<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: ty::InstanceDef<'tcx>,
 ) -> UnusedGenericParams {
+    assert!(instance.def_id().is_local());
+
     if !tcx.sess.opts.unstable_opts.polymorphize {
         // If polymorphization disabled, then all parameters are used.
         return UnusedGenericParams::new_all_used();
@@ -100,13 +102,6 @@ fn should_polymorphize<'tcx>(
         return false;
     }
 
-    // Polymorphization results are stored in cross-crate metadata only when there are unused
-    // parameters, so assume that non-local items must have only used parameters (else this query
-    // would not be invoked, and the cross-crate metadata used instead).
-    if !def_id.is_local() {
-        return false;
-    }
-
     // Foreign items have no bodies to analyze.
     if tcx.is_foreign_item(def_id) {
         return false;
diff --git a/compiler/rustc_parse/locales/en-US.ftl b/compiler/rustc_parse/messages.ftl
index e76e91fc1..f11d0ed0f 100644
--- a/compiler/rustc_parse/locales/en-US.ftl
+++ b/compiler/rustc_parse/messages.ftl
@@ -336,7 +336,7 @@ parse_expected_identifier_found_reserved_keyword = expected identifier, found re
 parse_expected_identifier_found_doc_comment = expected identifier, found doc comment
 parse_expected_identifier = expected identifier
 
-parse_sugg_escape_to_use_as_identifier = escape `{$ident_name}` to use it as an identifier
+parse_sugg_escape_identifier = escape `{$ident_name}` to use it as an identifier
 
 parse_sugg_remove_comma = remove this comma
 
@@ -412,8 +412,7 @@ parse_fn_ptr_with_generics = function pointer types may not have generic paramet
         *[false] a
     } `for` parameter list
 
-parse_invalid_identifier_with_leading_number = expected identifier, found number literal
-    .label = identifiers cannot start with a number
+parse_invalid_identifier_with_leading_number = identifiers cannot start with a number
 
 parse_maybe_fn_typo_with_impl = you might have meant to write `impl` instead of `fn`
     .suggestion = replace `fn` with `impl` here
@@ -710,7 +709,7 @@ parse_zero_chars = empty character literal
 parse_lone_slash = invalid trailing slash in literal
     .label = {parse_lone_slash}
 
-parse_unskipped_whitespace = non-ASCII whitespace symbol '{$ch}' is not skipped
+parse_unskipped_whitespace = whitespace symbol '{$ch}' is not skipped
     .label = {parse_unskipped_whitespace}
 
 parse_multiple_skipped_lines = multiple lines skipped by escaped newline
@@ -732,3 +731,14 @@ parse_unknown_start_of_token = unknown start of token: {$escaped}
         [one] once more
         *[other] {$repeats} more times
     }
+
+parse_box_syntax_removed = `box_syntax` has been removed
+    .suggestion = use `Box::new()` instead
+
+parse_bad_return_type_notation_output =
+    return type not allowed with return type notation
+    .suggestion = remove the return type
+
+parse_bad_return_type_notation_dotdot =
+    return type notation uses `()` instead of `(..)` for elided arguments
+    .suggestion = remove the `..`
diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 1662db36d..069217165 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -888,12 +888,12 @@ pub(crate) struct InvalidMetaItem {
 
 #[derive(Subdiagnostic)]
 #[suggestion(
-    parse_sugg_escape_to_use_as_identifier,
+    parse_sugg_escape_identifier,
     style = "verbose",
     applicability = "maybe-incorrect",
     code = "r#"
 )]
-pub(crate) struct SuggEscapeToUseAsIdentifier {
+pub(crate) struct SuggEscapeIdentifier {
     #[primary_span]
     pub span: Span,
     pub ident_name: String,
@@ -937,8 +937,9 @@ impl ExpectedIdentifierFound {
 pub(crate) struct ExpectedIdentifier {
     pub span: Span,
     pub token: Token,
-    pub suggest_raw: Option<SuggEscapeToUseAsIdentifier>,
+    pub suggest_raw: Option<SuggEscapeIdentifier>,
     pub suggest_remove_comma: Option<SuggRemoveComma>,
+    pub help_cannot_start_number: Option<HelpIdentifierStartsWithNumber>,
 }
 
 impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for ExpectedIdentifier {
@@ -975,10 +976,21 @@ impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for ExpectedIdentifier {
             sugg.add_to_diagnostic(&mut diag);
         }
 
+        if let Some(help) = self.help_cannot_start_number {
+            help.add_to_diagnostic(&mut diag);
+        }
+
         diag
     }
 }
 
+#[derive(Subdiagnostic)]
+#[help(parse_invalid_identifier_with_leading_number)]
+pub(crate) struct HelpIdentifierStartsWithNumber {
+    #[primary_span]
+    pub num_span: Span,
+}
+
 pub(crate) struct ExpectedSemi {
     pub span: Span,
     pub token: Token,
@@ -1208,14 +1220,6 @@ pub(crate) struct SelfParamNotFirst {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_invalid_identifier_with_leading_number)]
-pub(crate) struct InvalidIdentiferStartsWithNumber {
-    #[primary_span]
-    #[label]
-    pub span: Span,
-}
-
-#[derive(Diagnostic)]
 #[diag(parse_const_generic_without_braces)]
 pub(crate) struct ConstGenericWithoutBraces {
     #[primary_span]
@@ -2299,3 +2303,32 @@ impl HelpUseLatestEdition {
         }
     }
 }
+
+#[derive(Diagnostic)]
+#[diag(parse_box_syntax_removed)]
+pub struct BoxSyntaxRemoved<'a> {
+    #[primary_span]
+    #[suggestion(
+        code = "Box::new({code})",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
+    pub span: Span,
+    pub code: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_bad_return_type_notation_output)]
+pub(crate) struct BadReturnTypeNotationOutput {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_bad_return_type_notation_dotdot)]
+pub(crate) struct BadReturnTypeNotationDotDot {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub span: Span,
+}
diff --git a/compiler/rustc_parse/src/lexer/diagnostics.rs b/compiler/rustc_parse/src/lexer/diagnostics.rs
index 27f4428d3..9e6d27bf0 100644
--- a/compiler/rustc_parse/src/lexer/diagnostics.rs
+++ b/compiler/rustc_parse/src/lexer/diagnostics.rs
@@ -21,7 +21,7 @@ pub struct TokenTreeDiagInfo {
     pub matching_block_spans: Vec<(Span, Span)>,
 }
 
-pub fn same_identation_level(sm: &SourceMap, open_sp: Span, close_sp: Span) -> bool {
+pub fn same_indentation_level(sm: &SourceMap, open_sp: Span, close_sp: Span) -> bool {
     match (sm.span_to_margin(open_sp), sm.span_to_margin(close_sp)) {
         (Some(open_padding), Some(close_padding)) => open_padding == close_padding,
         _ => false,
@@ -67,13 +67,13 @@ pub fn report_suspicious_mismatch_block(
     let mut matched_spans: Vec<(Span, bool)> = diag_info
         .matching_block_spans
         .iter()
-        .map(|&(open, close)| (open.with_hi(close.lo()), same_identation_level(sm, open, close)))
+        .map(|&(open, close)| (open.with_hi(close.lo()), same_indentation_level(sm, open, close)))
         .collect();
 
     // sort by `lo`, so the large block spans in the front
-    matched_spans.sort_by(|a, b| a.0.lo().cmp(&b.0.lo()));
+    matched_spans.sort_by_key(|(span, _)| span.lo());
 
-    // We use larger block whose identation is well to cover those inner mismatched blocks
+    // We use larger block whose indentation is well to cover those inner mismatched blocks
     // O(N^2) here, but we are on error reporting path, so it is fine
     for i in 0..matched_spans.len() {
         let (block_span, same_ident) = matched_spans[i];
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
index 59958a309..9e856c9f2 100644
--- a/compiler/rustc_parse/src/lexer/mod.rs
+++ b/compiler/rustc_parse/src/lexer/mod.rs
@@ -175,7 +175,7 @@ impl<'a> StringReader<'a> {
                     if !sym.can_be_raw() {
                         self.sess.emit_err(errors::CannotBeRawIdent { span, ident: sym });
                     }
-                    self.sess.raw_identifier_spans.borrow_mut().push(span);
+                    self.sess.raw_identifier_spans.push(span);
                     token::Ident(sym, true)
                 }
                 rustc_lexer::TokenKind::UnknownPrefix => {
@@ -553,8 +553,8 @@ impl<'a> StringReader<'a> {
         }
 
         if let Some(possible_offset) = possible_offset {
-            let lo = start + BytePos(possible_offset as u32);
-            let hi = lo + BytePos(found_terminators as u32);
+            let lo = start + BytePos(possible_offset);
+            let hi = lo + BytePos(found_terminators);
             let span = self.mk_sp(lo, hi);
             err.span_suggestion(
                 span,
diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs
index 36fd1e37d..7c2c08951 100644
--- a/compiler/rustc_parse/src/lexer/tokentrees.rs
+++ b/compiler/rustc_parse/src/lexer/tokentrees.rs
@@ -1,5 +1,5 @@
 use super::diagnostics::report_suspicious_mismatch_block;
-use super::diagnostics::same_identation_level;
+use super::diagnostics::same_indentation_level;
 use super::diagnostics::TokenTreeDiagInfo;
 use super::{StringReader, UnmatchedDelim};
 use rustc_ast::token::{self, Delimiter, Token};
@@ -153,7 +153,7 @@ impl<'a> TokenTreesReader<'a> {
                         unclosed_delimiter = Some(sp);
                     };
                     for (brace, brace_span) in &self.diag_info.open_braces {
-                        if same_identation_level(&sm, self.token.span, *brace_span)
+                        if same_indentation_level(&sm, self.token.span, *brace_span)
                             && brace == &close_delim
                         {
                             // high likelihood of these two corresponding
diff --git a/compiler/rustc_parse/src/lexer/unicode_chars.rs b/compiler/rustc_parse/src/lexer/unicode_chars.rs
index d4f971d5b..1f027c08f 100644
--- a/compiler/rustc_parse/src/lexer/unicode_chars.rs
+++ b/compiler/rustc_parse/src/lexer/unicode_chars.rs
@@ -336,8 +336,8 @@ const ASCII_ARRAY: &[(&str, &str, Option<token::TokenKind>)] = &[
     ("\"", "Quotation Mark", None),
 ];
 
-pub(super) fn check_for_substitution<'a>(
-    reader: &StringReader<'a>,
+pub(super) fn check_for_substitution(
+    reader: &StringReader<'_>,
     pos: BytePos,
     ch: char,
     count: usize,
diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
index d1c3fd0cd..17466cd0e 100644
--- a/compiler/rustc_parse/src/lib.rs
+++ b/compiler/rustc_parse/src/lib.rs
@@ -36,7 +36,7 @@ pub mod validate_attr;
 
 mod errors;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 // A bunch of utility functions of the form `parse_<thing>_from_<source>`
 // where <thing> includes crate, expr, item, stmt, tts, and one that
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index a051dbe9f..e03ce5d71 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -6,20 +6,19 @@ use super::{
 use crate::errors::{
     AmbiguousPlus, AttributeOnParamType, BadQPathStage2, BadTypePlus, BadTypePlusSub,
     ComparisonOperatorsCannotBeChained, ComparisonOperatorsCannotBeChainedSugg,
-    ConstGenericWithoutBraces, ConstGenericWithoutBracesSugg, DocCommentOnParamType,
-    DoubleColonInBound, ExpectedIdentifier, ExpectedSemi, ExpectedSemiSugg,
-    GenericParamsWithoutAngleBrackets, GenericParamsWithoutAngleBracketsSugg, InInTypo,
-    IncorrectAwait, IncorrectSemicolon, IncorrectUseOfAwait, ParenthesesInForHead,
-    ParenthesesInForHeadSugg, PatternMethodParamWithoutBody, QuestionMarkInType,
-    QuestionMarkInTypeSugg, SelfParamNotFirst, StructLiteralBodyWithoutPath,
-    StructLiteralBodyWithoutPathSugg, StructLiteralNeedingParens, StructLiteralNeedingParensSugg,
-    SuggEscapeToUseAsIdentifier, SuggRemoveComma, UnexpectedConstInGenericParam,
-    UnexpectedConstParamDeclaration, UnexpectedConstParamDeclarationSugg, UnmatchedAngleBrackets,
-    UseEqInstead,
+    ConstGenericWithoutBraces, ConstGenericWithoutBracesSugg, DocCommentDoesNotDocumentAnything,
+    DocCommentOnParamType, DoubleColonInBound, ExpectedIdentifier, ExpectedSemi, ExpectedSemiSugg,
+    GenericParamsWithoutAngleBrackets, GenericParamsWithoutAngleBracketsSugg,
+    HelpIdentifierStartsWithNumber, InInTypo, IncorrectAwait, IncorrectSemicolon,
+    IncorrectUseOfAwait, ParenthesesInForHead, ParenthesesInForHeadSugg,
+    PatternMethodParamWithoutBody, QuestionMarkInType, QuestionMarkInTypeSugg, SelfParamNotFirst,
+    StructLiteralBodyWithoutPath, StructLiteralBodyWithoutPathSugg, StructLiteralNeedingParens,
+    StructLiteralNeedingParensSugg, SuggEscapeIdentifier, SuggRemoveComma,
+    UnexpectedConstInGenericParam, UnexpectedConstParamDeclaration,
+    UnexpectedConstParamDeclarationSugg, UnmatchedAngleBrackets, UseEqInstead,
 };
 
 use crate::fluent_generated as fluent;
-use crate::lexer::UnmatchedDelim;
 use crate::parser;
 use rustc_ast as ast;
 use rustc_ast::ptr::P;
@@ -39,7 +38,7 @@ use rustc_errors::{
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{kw, sym, Ident};
-use rustc_span::{Span, SpanSnippetError, DUMMY_SP};
+use rustc_span::{Span, SpanSnippetError, Symbol, DUMMY_SP};
 use std::mem::take;
 use std::ops::{Deref, DerefMut};
 use thin_vec::{thin_vec, ThinVec};
@@ -220,7 +219,6 @@ impl MultiSugg {
 /// is dropped.
 pub struct SnapshotParser<'a> {
     parser: Parser<'a>,
-    unclosed_delims: Vec<UnmatchedDelim>,
 }
 
 impl<'a> Deref for SnapshotParser<'a> {
@@ -255,34 +253,36 @@ impl<'a> Parser<'a> {
         &self.sess.span_diagnostic
     }
 
-    /// Replace `self` with `snapshot.parser` and extend `unclosed_delims` with `snapshot.unclosed_delims`.
-    /// This is to avoid losing unclosed delims errors `create_snapshot_for_diagnostic` clears.
+    /// Replace `self` with `snapshot.parser`.
     pub(super) fn restore_snapshot(&mut self, snapshot: SnapshotParser<'a>) {
         *self = snapshot.parser;
-        self.unclosed_delims.extend(snapshot.unclosed_delims);
-    }
-
-    pub fn unclosed_delims(&self) -> &[UnmatchedDelim] {
-        &self.unclosed_delims
     }
 
     /// Create a snapshot of the `Parser`.
     pub fn create_snapshot_for_diagnostic(&self) -> SnapshotParser<'a> {
-        let mut snapshot = self.clone();
-        let unclosed_delims = self.unclosed_delims.clone();
-        // Clear `unclosed_delims` in snapshot to avoid
-        // duplicate errors being emitted when the `Parser`
-        // is dropped (which may or may not happen, depending
-        // if the parsing the snapshot is created for is successful)
-        snapshot.unclosed_delims.clear();
-        SnapshotParser { parser: snapshot, unclosed_delims }
+        let snapshot = self.clone();
+        SnapshotParser { parser: snapshot }
     }
 
     pub(super) fn span_to_snippet(&self, span: Span) -> Result<String, SpanSnippetError> {
         self.sess.source_map().span_to_snippet(span)
     }
 
-    pub(super) fn expected_ident_found(&mut self) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+    /// Emits an error with suggestions if an identifier was expected but not found.
+    ///
+    /// Returns a possibly recovered identifier.
+    pub(super) fn expected_ident_found(
+        &mut self,
+        recover: bool,
+    ) -> PResult<'a, (Ident, /* is_raw */ bool)> {
+        if let TokenKind::DocComment(..) = self.prev_token.kind {
+            return Err(DocCommentDoesNotDocumentAnything {
+                span: self.prev_token.span,
+                missing_comma: None,
+            }
+            .into_diagnostic(&self.sess.span_diagnostic));
+        }
+
         let valid_follow = &[
             TokenKind::Eq,
             TokenKind::Colon,
@@ -294,38 +294,61 @@ impl<'a> Parser<'a> {
             TokenKind::CloseDelim(Delimiter::Brace),
             TokenKind::CloseDelim(Delimiter::Parenthesis),
         ];
-        let suggest_raw = match self.token.ident() {
-            Some((ident, false))
-                if ident.is_raw_guess()
-                    && self.look_ahead(1, |t| valid_follow.contains(&t.kind)) =>
-            {
-                Some(SuggEscapeToUseAsIdentifier {
-                    span: ident.span.shrink_to_lo(),
-                    // `Symbol::to_string()` is different from `Symbol::into_diagnostic_arg()`,
-                    // which uses `Symbol::to_ident_string()` and "helpfully" adds an implicit `r#`
-                    ident_name: ident.name.to_string(),
-                })
-            }
-            _ => None,
-        };
+
+        let mut recovered_ident = None;
+        // we take this here so that the correct original token is retained in
+        // the diagnostic, regardless of eager recovery.
+        let bad_token = self.token.clone();
+
+        // suggest prepending a keyword in identifier position with `r#`
+        let suggest_raw = if let Some((ident, false)) = self.token.ident()
+            && ident.is_raw_guess()
+            && self.look_ahead(1, |t| valid_follow.contains(&t.kind))
+        {
+            recovered_ident = Some((ident, true));
+
+            // `Symbol::to_string()` is different from `Symbol::into_diagnostic_arg()`,
+            // which uses `Symbol::to_ident_string()` and "helpfully" adds an implicit `r#`
+            let ident_name = ident.name.to_string();
+
+            Some(SuggEscapeIdentifier {
+                span: ident.span.shrink_to_lo(),
+                ident_name
+            })
+        } else { None };
 
         let suggest_remove_comma =
             if self.token == token::Comma && self.look_ahead(1, |t| t.is_ident()) {
-                Some(SuggRemoveComma { span: self.token.span })
+                if recover {
+                    self.bump();
+                    recovered_ident = self.ident_or_err(false).ok();
+                };
+
+                Some(SuggRemoveComma { span: bad_token.span })
             } else {
                 None
             };
 
+        let help_cannot_start_number = self.is_lit_bad_ident().map(|(len, valid_portion)| {
+            let (invalid, valid) = self.token.span.split_at(len as u32);
+
+            recovered_ident = Some((Ident::new(valid_portion, valid), false));
+
+            HelpIdentifierStartsWithNumber { num_span: invalid }
+        });
+
         let err = ExpectedIdentifier {
-            span: self.token.span,
-            token: self.token.clone(),
+            span: bad_token.span,
+            token: bad_token,
             suggest_raw,
             suggest_remove_comma,
+            help_cannot_start_number,
         };
         let mut err = err.into_diagnostic(&self.sess.span_diagnostic);
 
         // if the token we have is a `<`
         // it *might* be a misplaced generic
+        // FIXME: could we recover with this?
         if self.token == token::Lt {
             // all keywords that could have generic applied
             let valid_prev_keywords =
@@ -376,7 +399,38 @@ impl<'a> Parser<'a> {
             }
         }
 
-        err
+        if let Some(recovered_ident) = recovered_ident && recover {
+            err.emit();
+            Ok(recovered_ident)
+        } else {
+            Err(err)
+        }
+    }
+
+    pub(super) fn expected_ident_found_err(&mut self) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+        self.expected_ident_found(false).unwrap_err()
+    }
+
+    /// Checks if the current token is a integer or float literal and looks like
+    /// it could be a invalid identifier with digits at the start.
+    ///
+    /// Returns the number of characters (bytes) composing the invalid portion
+    /// of the identifier and the valid portion of the identifier.
+    pub(super) fn is_lit_bad_ident(&mut self) -> Option<(usize, Symbol)> {
+        // ensure that the integer literal is followed by a *invalid*
+        // suffix: this is how we know that it is a identifier with an
+        // invalid beginning.
+        if let token::Literal(Lit {
+            kind: token::LitKind::Integer | token::LitKind::Float,
+            symbol,
+            suffix: Some(suffix), // no suffix makes it a valid literal
+        }) = self.token.kind
+            && rustc_ast::MetaItemLit::from_token(&self.token).is_none()
+        {
+            Some((symbol.as_str().len(), suffix))
+        } else {
+            None
+        }
     }
 
     pub(super) fn expected_one_of_not_found(
@@ -579,21 +633,6 @@ impl<'a> Parser<'a> {
         } else {
             label_sp
         };
-        match self.recover_closing_delimiter(
-            &expected
-                .iter()
-                .filter_map(|tt| match tt {
-                    TokenType::Token(t) => Some(t.clone()),
-                    _ => None,
-                })
-                .collect::<Vec<_>>(),
-            err,
-        ) {
-            Err(e) => err = e,
-            Ok(recovered) => {
-                return Ok(recovered);
-            }
-        }
 
         if self.check_too_many_raw_str_terminators(&mut err) {
             if expected.contains(&TokenType::Token(token::Semi)) && self.eat(&token::Semi) {
@@ -950,8 +989,7 @@ impl<'a> Parser<'a> {
                     }
                     if self.token.kind == token::OpenDelim(Delimiter::Parenthesis) {
                         // Recover from bad turbofish: `foo.collect::Vec<_>()`.
-                        let args = AngleBracketedArgs { args, span }.into();
-                        segment.args = args;
+                        segment.args = Some(AngleBracketedArgs { args, span }.into());
 
                         self.sess.emit_err(GenericParamsWithoutAngleBrackets {
                             span,
@@ -1573,12 +1611,6 @@ impl<'a> Parser<'a> {
         );
         let mut err = self.struct_span_err(sp, &msg);
         let label_exp = format!("expected `{token_str}`");
-        match self.recover_closing_delimiter(&[t.clone()], err) {
-            Err(e) => err = e,
-            Ok(recovered) => {
-                return Ok(recovered);
-            }
-        }
         let sm = self.sess.source_map();
         if !sm.is_multiline(prev_sp.until(sp)) {
             // When the spans are in the same line, it means that the only content
@@ -1795,81 +1827,6 @@ impl<'a> Parser<'a> {
         }
     }
 
-    pub(super) fn recover_closing_delimiter(
-        &mut self,
-        tokens: &[TokenKind],
-        mut err: DiagnosticBuilder<'a, ErrorGuaranteed>,
-    ) -> PResult<'a, bool> {
-        let mut pos = None;
-        // We want to use the last closing delim that would apply.
-        for (i, unmatched) in self.unclosed_delims.iter().enumerate().rev() {
-            if tokens.contains(&token::CloseDelim(unmatched.expected_delim))
-                && Some(self.token.span) > unmatched.unclosed_span
-            {
-                pos = Some(i);
-            }
-        }
-        match pos {
-            Some(pos) => {
-                // Recover and assume that the detected unclosed delimiter was meant for
-                // this location. Emit the diagnostic and act as if the delimiter was
-                // present for the parser's sake.
-
-                // Don't attempt to recover from this unclosed delimiter more than once.
-                let unmatched = self.unclosed_delims.remove(pos);
-                let delim = TokenType::Token(token::CloseDelim(unmatched.expected_delim));
-                if unmatched.found_delim.is_none() {
-                    // We encountered `Eof`, set this fact here to avoid complaining about missing
-                    // `fn main()` when we found place to suggest the closing brace.
-                    *self.sess.reached_eof.borrow_mut() = true;
-                }
-
-                // We want to suggest the inclusion of the closing delimiter where it makes
-                // the most sense, which is immediately after the last token:
-                //
-                //  {foo(bar {}}
-                //      ^      ^
-                //      |      |
-                //      |      help: `)` may belong here
-                //      |
-                //      unclosed delimiter
-                if let Some(sp) = unmatched.unclosed_span {
-                    let mut primary_span: Vec<Span> =
-                        err.span.primary_spans().iter().cloned().collect();
-                    primary_span.push(sp);
-                    let mut primary_span: MultiSpan = primary_span.into();
-                    for span_label in err.span.span_labels() {
-                        if let Some(label) = span_label.label {
-                            primary_span.push_span_label(span_label.span, label);
-                        }
-                    }
-                    err.set_span(primary_span);
-                    err.span_label(sp, "unclosed delimiter");
-                }
-                // Backticks should be removed to apply suggestions.
-                let mut delim = delim.to_string();
-                delim.retain(|c| c != '`');
-                err.span_suggestion_short(
-                    self.prev_token.span.shrink_to_hi(),
-                    &format!("`{delim}` may belong here"),
-                    delim,
-                    Applicability::MaybeIncorrect,
-                );
-                if unmatched.found_delim.is_none() {
-                    // Encountered `Eof` when lexing blocks. Do not recover here to avoid knockdown
-                    // errors which would be emitted elsewhere in the parser and let other error
-                    // recovery consume the rest of the file.
-                    Err(err)
-                } else {
-                    err.emit();
-                    self.expected_tokens.clear(); // Reduce the number of errors.
-                    Ok(true)
-                }
-            }
-            _ => Err(err),
-        }
-    }
-
     /// Eats tokens until we can be relatively sure we reached the end of the
     /// statement. This is something of a best-effort heuristic.
     ///
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 95a7ca80d..03c82fbd3 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -8,6 +8,7 @@ use super::{
 
 use crate::errors;
 use crate::maybe_recover_from_interpolated_ty_qpath;
+use ast::{Path, PathSegment};
 use core::mem;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
@@ -29,6 +30,7 @@ use rustc_session::errors::{report_lit_error, ExprParenthesesNeeded};
 use rustc_session::lint::builtin::BREAK_WITH_LABEL_AND_LOOP;
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_span::source_map::{self, Span, Spanned};
+use rustc_span::symbol::kw::PathRoot;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{BytePos, Pos};
 use thin_vec::{thin_vec, ThinVec};
@@ -636,11 +638,27 @@ impl<'a> Parser<'a> {
         self.parse_expr_unary(lo, UnOp::Not)
     }
 
-    /// Parse `box expr`.
+    /// Parse `box expr` - this syntax has been removed, but we still parse this
+    /// for now to provide an automated way to fix usages of it
     fn parse_expr_box(&mut self, lo: Span) -> PResult<'a, (Span, ExprKind)> {
         let (span, expr) = self.parse_expr_prefix_common(lo)?;
-        self.sess.gated_spans.gate(sym::box_syntax, span);
-        Ok((span, ExprKind::Box(expr)))
+        let code = self.sess.source_map().span_to_snippet(span.with_lo(lo.hi())).unwrap();
+        self.sess.emit_err(errors::BoxSyntaxRemoved { span, code: code.trim() });
+        // So typechecking works, parse `box <expr>` as `::std::boxed::Box::new(expr)`
+        let path = Path {
+            span,
+            segments: [
+                PathSegment::from_ident(Ident::with_dummy_span(PathRoot)),
+                PathSegment::from_ident(Ident::with_dummy_span(sym::std)),
+                PathSegment::from_ident(Ident::from_str("boxed")),
+                PathSegment::from_ident(Ident::from_str("Box")),
+                PathSegment::from_ident(Ident::with_dummy_span(sym::new)),
+            ]
+            .into(),
+            tokens: None,
+        };
+        let path = self.mk_expr(span, ExprKind::Path(None, path));
+        Ok((span, self.mk_call(path, ThinVec::from([expr]))))
     }
 
     fn is_mistaken_not_ident_negation(&self) -> bool {
@@ -1394,19 +1412,6 @@ impl<'a> Parser<'a> {
             self.parse_expr_let()
         } else if self.eat_keyword(kw::Underscore) {
             Ok(self.mk_expr(self.prev_token.span, ExprKind::Underscore))
-        } else if !self.unclosed_delims.is_empty() && self.check(&token::Semi) {
-            // Don't complain about bare semicolons after unclosed braces
-            // recovery in order to keep the error count down. Fixing the
-            // delimiters will possibly also fix the bare semicolon found in
-            // expression context. For example, silence the following error:
-            //
-            //     error: expected expression, found `;`
-            //      --> file.rs:2:13
-            //       |
-            //     2 |     foo(bar(;
-            //       |             ^ expected expression
-            self.bump();
-            Ok(self.mk_expr_err(self.token.span))
         } else if self.token.uninterpolated_span().rust_2018() {
             // `Span::rust_2018()` is somewhat expensive; don't get it repeatedly.
             if self.check_keyword(kw::Async) {
@@ -1838,20 +1843,14 @@ impl<'a> Parser<'a> {
         &mut self,
         mk_lit_char: impl FnOnce(Symbol, Span) -> L,
     ) -> PResult<'a, L> {
-        if let token::Interpolated(inner) = &self.token.kind {
-            let expr = match inner.as_ref() {
-                token::NtExpr(expr) => Some(expr),
-                token::NtLiteral(expr) => Some(expr),
-                _ => None,
-            };
-            if let Some(expr) = expr {
-                if matches!(expr.kind, ExprKind::Err) {
-                    let mut err = errors::InvalidInterpolatedExpression { span: self.token.span }
-                        .into_diagnostic(&self.sess.span_diagnostic);
-                    err.downgrade_to_delayed_bug();
-                    return Err(err);
-                }
-            }
+        if let token::Interpolated(nt) = &self.token.kind
+            && let token::NtExpr(e) | token::NtLiteral(e) = &**nt
+            && matches!(e.kind, ExprKind::Err)
+        {
+            let mut err = errors::InvalidInterpolatedExpression { span: self.token.span }
+                .into_diagnostic(&self.sess.span_diagnostic);
+            err.downgrade_to_delayed_bug();
+            return Err(err);
         }
         let token = self.token.clone();
         let err = |self_: &Self| {
@@ -2118,7 +2117,7 @@ impl<'a> Parser<'a> {
             ClosureBinder::NotPresent
         };
 
-        let constness = self.parse_closure_constness(Case::Sensitive);
+        let constness = self.parse_closure_constness();
 
         let movability =
             if self.eat_keyword(kw::Static) { Movability::Static } else { Movability::Movable };
@@ -2768,7 +2767,7 @@ impl<'a> Parser<'a> {
                     (token::DotDotEq, token::Gt)
                 ) {
                     // `error_inclusive_range_match_arrow` handles cases like `0..=> {}`,
-                    // so we supress the error here
+                    // so we suppress the error here
                     err.delay_as_bug();
                     this.bump();
                 } else {
@@ -2912,7 +2911,7 @@ impl<'a> Parser<'a> {
         self.expect_keyword(kw::Async)?;
         let capture_clause = self.parse_capture_clause()?;
         let (attrs, body) = self.parse_inner_attrs_and_block()?;
-        let kind = ExprKind::Async(capture_clause, DUMMY_NODE_ID, body);
+        let kind = ExprKind::Async(capture_clause, body);
         Ok(self.mk_expr_with_attrs(lo.to(self.prev_token.span), kind, attrs))
     }
 
diff --git a/compiler/rustc_parse/src/parser/generics.rs b/compiler/rustc_parse/src/parser/generics.rs
index 8d0f168e0..f8ef1307c 100644
--- a/compiler/rustc_parse/src/parser/generics.rs
+++ b/compiler/rustc_parse/src/parser/generics.rs
@@ -53,7 +53,7 @@ impl<'a> Parser<'a> {
                 let snapshot = self.create_snapshot_for_diagnostic();
                 match self.parse_ty() {
                     Ok(p) => {
-                        if let TyKind::ImplTrait(_, bounds) = &(*p).kind {
+                        if let TyKind::ImplTrait(_, bounds) = &p.kind {
                             let span = impl_span.to(self.token.span.shrink_to_lo());
                             let mut err = self.struct_span_err(
                                 span,
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 9d9ae154a..6422b8ac1 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -3,6 +3,7 @@ use crate::errors;
 use super::diagnostics::{dummy_arg, ConsumeClosingDelim};
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
 use super::{AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, TrailingToken};
+use ast::StaticItem;
 use rustc_ast::ast::*;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, TokenKind};
@@ -125,16 +126,13 @@ impl<'a> Parser<'a> {
             return Ok(Some(item.into_inner()));
         };
 
-        let mut unclosed_delims = vec![];
         let item =
             self.collect_tokens_trailing_token(attrs, force_collect, |this: &mut Self, attrs| {
                 let item =
                     this.parse_item_common_(attrs, mac_allowed, attrs_allowed, fn_parse_mode);
-                unclosed_delims.append(&mut this.unclosed_delims);
                 Ok((item?, TrailingToken::None))
             })?;
 
-        self.unclosed_delims.append(&mut unclosed_delims);
         Ok(item)
     }
 
@@ -230,7 +228,7 @@ impl<'a> Parser<'a> {
             self.bump(); // `static`
             let m = self.parse_mutability();
             let (ident, ty, expr) = self.parse_item_global(Some(m))?;
-            (ident, ItemKind::Static(ty, m, expr))
+            (ident, ItemKind::Static(Box::new(StaticItem { ty, mutability: m, expr })))
         } else if let Const::Yes(const_span) = self.parse_constness(Case::Sensitive) {
             // CONST ITEM
             if self.token.is_keyword(kw::Impl) {
@@ -239,7 +237,7 @@ impl<'a> Parser<'a> {
             } else {
                 self.recover_const_mut(const_span);
                 let (ident, ty, expr) = self.parse_item_global(None)?;
-                (ident, ItemKind::Const(def_(), ty, expr))
+                (ident, ItemKind::Const(Box::new(ConstItem { defaultness: def_(), ty, expr })))
             }
         } else if self.check_keyword(kw::Trait) || self.check_auto_or_unsafe_trait_item() {
             // TRAIT ITEM
@@ -865,9 +863,13 @@ impl<'a> Parser<'a> {
                 let kind = match AssocItemKind::try_from(kind) {
                     Ok(kind) => kind,
                     Err(kind) => match kind {
-                        ItemKind::Static(a, _, b) => {
+                        ItemKind::Static(box StaticItem { ty, mutability: _, expr }) => {
                             self.sess.emit_err(errors::AssociatedStaticItemNotAllowed { span });
-                            AssocItemKind::Const(Defaultness::Final, a, b)
+                            AssocItemKind::Const(Box::new(ConstItem {
+                                defaultness: Defaultness::Final,
+                                ty,
+                                expr,
+                            }))
                         }
                         _ => return self.error_bad_item_kind(span, &kind, "`trait`s or `impl`s"),
                     },
@@ -1117,12 +1119,12 @@ impl<'a> Parser<'a> {
                 let kind = match ForeignItemKind::try_from(kind) {
                     Ok(kind) => kind,
                     Err(kind) => match kind {
-                        ItemKind::Const(_, a, b) => {
+                        ItemKind::Const(box ConstItem { ty, expr, .. }) => {
                             self.sess.emit_err(errors::ExternItemCannotBeConst {
                                 ident_span: ident.span,
                                 const_span: span.with_hi(ident.span.lo()),
                             });
-                            ForeignItemKind::Static(a, Mutability::Not, b)
+                            ForeignItemKind::Static(ty, Mutability::Not, expr)
                         }
                         _ => return self.error_bad_item_kind(span, &kind, "`extern` blocks"),
                     },
@@ -1184,7 +1186,7 @@ impl<'a> Parser<'a> {
         defaultness: Defaultness,
     ) -> PResult<'a, ItemInfo> {
         let impl_span = self.token.span;
-        let mut err = self.expected_ident_found();
+        let mut err = self.expected_ident_found_err();
 
         // Only try to recover if this is implementing a trait for a type
         let mut impl_info = match self.parse_item_impl(attrs, defaultness) {
@@ -1747,7 +1749,7 @@ impl<'a> Parser<'a> {
     /// Parses a field identifier. Specialized version of `parse_ident_common`
     /// for better diagnostics and suggestions.
     fn parse_field_ident(&mut self, adt_ty: &str, lo: Span) -> PResult<'a, Ident> {
-        let (ident, is_raw) = self.ident_or_err()?;
+        let (ident, is_raw) = self.ident_or_err(true)?;
         if !is_raw && ident.is_reserved() {
             let snapshot = self.create_snapshot_for_diagnostic();
             let err = if self.check_fn_front_matter(false, Case::Sensitive) {
@@ -1779,7 +1781,7 @@ impl<'a> Parser<'a> {
                     Err(err) => {
                         err.cancel();
                         self.restore_snapshot(snapshot);
-                        self.expected_ident_found()
+                        self.expected_ident_found_err()
                     }
                 }
             } else if self.eat_keyword(kw::Struct) {
@@ -1795,11 +1797,11 @@ impl<'a> Parser<'a> {
                     Err(err) => {
                         err.cancel();
                         self.restore_snapshot(snapshot);
-                        self.expected_ident_found()
+                        self.expected_ident_found_err()
                     }
                 }
             } else {
-                let mut err = self.expected_ident_found();
+                let mut err = self.expected_ident_found_err();
                 if self.eat_keyword_noexpect(kw::Let)
                     && let removal_span = self.prev_token.span.until(self.token.span)
                     && let Ok(ident) = self.parse_ident_common(false)
@@ -1960,21 +1962,12 @@ impl<'a> Parser<'a> {
         // FIXME: This will make us not emit the help even for declarative
         // macros within the same crate (that we can fix), which is sad.
         if !span.from_expansion() {
-            if self.unclosed_delims.is_empty() {
-                let DelimSpan { open, close } = args.dspan;
-                err.multipart_suggestion(
-                    "change the delimiters to curly braces",
-                    vec![(open, "{".to_string()), (close, '}'.to_string())],
-                    Applicability::MaybeIncorrect,
-                );
-            } else {
-                err.span_suggestion(
-                    span,
-                    "change the delimiters to curly braces",
-                    " { /* items */ }",
-                    Applicability::HasPlaceholders,
-                );
-            }
+            let DelimSpan { open, close } = args.dspan;
+            err.multipart_suggestion(
+                "change the delimiters to curly braces",
+                vec![(open, "{".to_string()), (close, '}'.to_string())],
+                Applicability::MaybeIncorrect,
+            );
             err.span_suggestion(
                 span.shrink_to_hi(),
                 "add a semicolon",
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index da82e4724..aa57b8047 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -29,6 +29,7 @@ use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, MacDelimiter, Mutab
 use rustc_ast::{HasAttrs, HasTokens, Unsafe, Visibility, VisibilityKind};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::Ordering;
 use rustc_errors::PResult;
 use rustc_errors::{
     Applicability, DiagnosticBuilder, ErrorGuaranteed, FatalError, IntoDiagnostic, MultiSpan,
@@ -42,8 +43,7 @@ use thin_vec::ThinVec;
 use tracing::debug;
 
 use crate::errors::{
-    DocCommentDoesNotDocumentAnything, IncorrectVisibilityRestriction, MismatchedClosingDelimiter,
-    NonStringAbiLiteral,
+    IncorrectVisibilityRestriction, MismatchedClosingDelimiter, NonStringAbiLiteral,
 };
 
 bitflags::bitflags! {
@@ -146,10 +146,7 @@ pub struct Parser<'a> {
     /// See the comments in the `parse_path_segment` function for more details.
     unmatched_angle_bracket_count: u32,
     max_angle_bracket_count: u32,
-    /// A list of all unclosed delimiters found by the lexer. If an entry is used for error recovery
-    /// it gets removed from here. Every entry left at the end gets emitted as an independent
-    /// error.
-    pub(super) unclosed_delims: Vec<UnmatchedDelim>,
+
     last_unexpected_token_span: Option<Span>,
     /// Span pointing at the `:` for the last type ascription the parser has seen, and whether it
     /// looked like it could have been a mistyped path or literal `Option:Some(42)`).
@@ -168,7 +165,7 @@ pub struct Parser<'a> {
 // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
 // it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(Parser<'_>, 312);
+rustc_data_structures::static_assert_size!(Parser<'_>, 288);
 
 /// Stores span information about a closure.
 #[derive(Clone)]
@@ -215,12 +212,6 @@ struct CaptureState {
     inner_attr_ranges: FxHashMap<AttrId, ReplaceRange>,
 }
 
-impl<'a> Drop for Parser<'a> {
-    fn drop(&mut self) {
-        emit_unclosed_delims(&mut self.unclosed_delims, &self.sess);
-    }
-}
-
 /// Iterator over a `TokenStream` that produces `Token`s. It's a bit odd that
 /// we (a) lex tokens into a nice tree structure (`TokenStream`), and then (b)
 /// use this type to emit them as a linear sequence. But a linear sequence is
@@ -478,7 +469,6 @@ impl<'a> Parser<'a> {
             desugar_doc_comments,
             unmatched_angle_bracket_count: 0,
             max_angle_bracket_count: 0,
-            unclosed_delims: Vec::new(),
             last_unexpected_token_span: None,
             last_type_ascription: None,
             subparser_name,
@@ -562,21 +552,11 @@ impl<'a> Parser<'a> {
         self.parse_ident_common(true)
     }
 
-    fn ident_or_err(&mut self) -> PResult<'a, (Ident, /* is_raw */ bool)> {
-        self.token.ident().ok_or_else(|| match self.prev_token.kind {
-            TokenKind::DocComment(..) => DocCommentDoesNotDocumentAnything {
-                span: self.prev_token.span,
-                missing_comma: None,
-            }
-            .into_diagnostic(&self.sess.span_diagnostic),
-            _ => self.expected_ident_found(),
-        })
-    }
-
     fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, Ident> {
-        let (ident, is_raw) = self.ident_or_err()?;
+        let (ident, is_raw) = self.ident_or_err(recover)?;
+
         if !is_raw && ident.is_reserved() {
-            let mut err = self.expected_ident_found();
+            let mut err = self.expected_ident_found_err();
             if recover {
                 err.emit();
             } else {
@@ -587,6 +567,21 @@ impl<'a> Parser<'a> {
         Ok(ident)
     }
 
+    fn ident_or_err(&mut self, recover: bool) -> PResult<'a, (Ident, /* is_raw */ bool)> {
+        let result = self.token.ident().ok_or_else(|| self.expected_ident_found(recover));
+
+        let (ident, is_raw) = match result {
+            Ok(ident) => ident,
+            Err(err) => match err {
+                // we recovered!
+                Ok(ident) => ident,
+                Err(err) => return Err(err),
+            },
+        };
+
+        Ok((ident, is_raw))
+    }
+
     /// Checks if the next token is `tok`, and returns `true` if so.
     ///
     /// This method will automatically add `tok` to `expected_tokens` if `tok` is not
@@ -859,7 +854,6 @@ impl<'a> Parser<'a> {
         let mut recovered = false;
         let mut trailing = false;
         let mut v = ThinVec::new();
-        let unclosed_delims = !self.unclosed_delims.is_empty();
 
         while !self.expect_any_with_type(kets, expect) {
             if let token::CloseDelim(..) | token::Eof = self.token.kind {
@@ -901,7 +895,7 @@ impl<'a> Parser<'a> {
                                 _ => {
                                     // Attempt to keep parsing if it was a similar separator.
                                     if let Some(tokens) = t.similar_tokens() {
-                                        if tokens.contains(&self.token.kind) && !unclosed_delims {
+                                        if tokens.contains(&self.token.kind) {
                                             self.bump();
                                         }
                                     }
@@ -1207,9 +1201,13 @@ impl<'a> Parser<'a> {
         self.parse_constness_(case, false)
     }
 
-    /// Parses constness for closures
-    fn parse_closure_constness(&mut self, case: Case) -> Const {
-        self.parse_constness_(case, true)
+    /// Parses constness for closures (case sensitive, feature-gated)
+    fn parse_closure_constness(&mut self) -> Const {
+        let constness = self.parse_constness_(Case::Sensitive, true);
+        if let Const::Yes(span) = constness {
+            self.sess.gated_spans.gate(sym::const_closures, span);
+        }
+        constness
     }
 
     fn parse_constness_(&mut self, case: Case, is_closure: bool) -> Const {
@@ -1543,8 +1541,10 @@ pub(crate) fn make_unclosed_delims_error(
 }
 
 pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedDelim>, sess: &ParseSess) {
-    *sess.reached_eof.borrow_mut() |=
-        unclosed_delims.iter().any(|unmatched_delim| unmatched_delim.found_delim.is_none());
+    let _ = sess.reached_eof.fetch_or(
+        unclosed_delims.iter().any(|unmatched_delim| unmatched_delim.found_delim.is_none()),
+        Ordering::Relaxed,
+    );
     for unmatched in unclosed_delims.drain(..) {
         if let Some(mut e) = make_unclosed_delims_error(unmatched, sess) {
             e.emit();
diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index 8e920f1c4..2246002f5 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -391,7 +391,13 @@ impl<'a> Parser<'a> {
             } else {
                 PatKind::Lit(const_expr)
             }
-        } else if self.can_be_ident_pat() {
+        // Don't eagerly error on semantically invalid tokens when matching
+        // declarative macros, as the input to those doesn't have to be
+        // semantically valid. For attribute/derive proc macros this is not the
+        // case, so doing the recovery for them is fine.
+        } else if self.can_be_ident_pat()
+            || (self.is_lit_bad_ident().is_some() && self.may_recover())
+        {
             // Parse `ident @ pat`
             // This can give false positives and parse nullary enums,
             // they are dealt with later in resolve.
@@ -590,7 +596,7 @@ impl<'a> Parser<'a> {
         // Make sure we don't allow e.g. `let mut $p;` where `$p:pat`.
         if let token::Interpolated(nt) = &self.token.kind {
             if let token::NtPat(_) = **nt {
-                self.expected_ident_found().emit();
+                self.expected_ident_found_err().emit();
             }
         }
 
diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs
index b50d2984a..c25c23d84 100644
--- a/compiler/rustc_parse/src/parser/path.rs
+++ b/compiler/rustc_parse/src/parser/path.rs
@@ -1,6 +1,6 @@
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
 use super::{Parser, Restrictions, TokenType};
-use crate::maybe_whole;
+use crate::{errors, maybe_whole};
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
 use rustc_ast::{
@@ -290,6 +290,32 @@ impl<'a> Parser<'a> {
                     })?;
                     let span = lo.to(self.prev_token.span);
                     AngleBracketedArgs { args, span }.into()
+                } else if self.may_recover()
+                    && self.token.kind == token::OpenDelim(Delimiter::Parenthesis)
+                    // FIXME(return_type_notation): Could also recover `...` here.
+                    && self.look_ahead(1, |tok| tok.kind == token::DotDot)
+                {
+                    self.bump();
+                    self.sess
+                        .emit_err(errors::BadReturnTypeNotationDotDot { span: self.token.span });
+                    self.bump();
+                    self.expect(&token::CloseDelim(Delimiter::Parenthesis))?;
+                    let span = lo.to(self.prev_token.span);
+
+                    if self.eat_noexpect(&token::RArrow) {
+                        let lo = self.prev_token.span;
+                        let ty = self.parse_ty()?;
+                        self.sess
+                            .emit_err(errors::BadReturnTypeNotationOutput { span: lo.to(ty.span) });
+                    }
+
+                    ParenthesizedArgs {
+                        span,
+                        inputs: ThinVec::new(),
+                        inputs_span: span,
+                        output: ast::FnRetTy::Default(self.prev_token.span.shrink_to_hi()),
+                    }
+                    .into()
                 } else {
                     // `(T, U) -> R`
                     let (inputs, _) = self.parse_paren_comma_seq(|p| p.parse_ty())?;
@@ -300,7 +326,7 @@ impl<'a> Parser<'a> {
                     ParenthesizedArgs { span, inputs, inputs_span, output }.into()
                 };
 
-                PathSegment { ident, args, id: ast::DUMMY_NODE_ID }
+                PathSegment { ident, args: Some(args), id: ast::DUMMY_NODE_ID }
             } else {
                 // Generic arguments are not found.
                 PathSegment::from_ident(ident)
@@ -547,10 +573,16 @@ impl<'a> Parser<'a> {
                     };
 
                     let span = lo.to(self.prev_token.span);
-
                     // Gate associated type bounds, e.g., `Iterator<Item: Ord>`.
                     if let AssocConstraintKind::Bound { .. } = kind {
-                        self.sess.gated_spans.gate(sym::associated_type_bounds, span);
+                        if let Some(ast::GenericArgs::Parenthesized(args)) = &gen_args
+                            && args.inputs.is_empty()
+                            && matches!(args.output, ast::FnRetTy::Default(..))
+                        {
+                            self.sess.gated_spans.gate(sym::return_type_notation, span);
+                        } else {
+                            self.sess.gated_spans.gate(sym::associated_type_bounds, span);
+                        }
                     }
                     let constraint =
                         AssocConstraint { id: ast::DUMMY_NODE_ID, ident, gen_args, kind, span };
diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs
index 92a22ffc2..fbe5b88c4 100644
--- a/compiler/rustc_parse/src/parser/stmt.rs
+++ b/compiler/rustc_parse/src/parser/stmt.rs
@@ -273,7 +273,6 @@ impl<'a> Parser<'a> {
             self.bump();
         }
 
-        self.report_invalid_identifier_error()?;
         let (pat, colon) =
             self.parse_pat_before_ty(None, RecoverComma::Yes, PatternLocation::LetBinding)?;
 
@@ -366,17 +365,6 @@ impl<'a> Parser<'a> {
         Ok(P(ast::Local { ty, pat, kind, id: DUMMY_NODE_ID, span: lo.to(hi), attrs, tokens: None }))
     }
 
-    /// report error for `let 1x = 123`
-    pub fn report_invalid_identifier_error(&mut self) -> PResult<'a, ()> {
-        if let token::Literal(lit) = self.token.uninterpolate().kind &&
-            rustc_ast::MetaItemLit::from_token(&self.token).is_none() &&
-            (lit.kind == token::LitKind::Integer || lit.kind == token::LitKind::Float) &&
-            self.look_ahead(1, |t| matches!(t.kind, token::Eq) || matches!(t.kind, token::Colon ) ) {
-                return Err(self.sess.create_err(errors::InvalidIdentiferStartsWithNumber { span: self.token.span }));
-        }
-        Ok(())
-    }
-
     fn check_let_else_init_bool_expr(&self, init: &ast::Expr) {
         if let ast::ExprKind::Binary(op, ..) = init.kind {
             if op.node.lazy() {
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index 6fe4da71f..400c8dbe9 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -624,10 +624,12 @@ impl<'a> Parser<'a> {
     ///
     /// Note that this does *not* parse bare trait objects.
     fn parse_dyn_ty(&mut self, impl_dyn_multi: &mut bool) -> PResult<'a, TyKind> {
+        let lo = self.token.span;
         self.bump(); // `dyn`
 
         // parse dyn* types
         let syntax = if self.eat(&TokenKind::BinOp(token::Star)) {
+            self.sess.gated_spans.gate(sym::dyn_star, lo.to(self.prev_token.span));
             TraitObjectSyntax::DynStar
         } else {
             TraitObjectSyntax::Dyn
@@ -1057,8 +1059,11 @@ impl<'a> Parser<'a> {
             output,
         }
         .into();
-        *fn_path_segment =
-            ast::PathSegment { ident: fn_path_segment.ident, args, id: ast::DUMMY_NODE_ID };
+        *fn_path_segment = ast::PathSegment {
+            ident: fn_path_segment.ident,
+            args: Some(args),
+            id: ast::DUMMY_NODE_ID,
+        };
 
         // Convert parsed `<'a>` in `Fn<'a>` into `for<'a>`.
         let mut generic_params = lifetimes
diff --git a/compiler/rustc_parse_format/src/lib.rs b/compiler/rustc_parse_format/src/lib.rs
index 8a3cedfee..7de84db21 100644
--- a/compiler/rustc_parse_format/src/lib.rs
+++ b/compiler/rustc_parse_format/src/lib.rs
@@ -14,6 +14,7 @@
 // We want to be able to build this crate with a stable compiler, so no
 // `#![feature]` attributes should be added.
 
+use rustc_lexer::unescape;
 pub use Alignment::*;
 pub use Count::*;
 pub use Piece::*;
@@ -234,8 +235,10 @@ pub struct Parser<'a> {
     last_opening_brace: Option<InnerSpan>,
     /// Whether the source string is comes from `println!` as opposed to `format!` or `print!`
     append_newline: bool,
-    /// Whether this formatting string is a literal or it comes from a macro.
-    pub is_literal: bool,
+    /// Whether this formatting string was written directly in the source. This controls whether we
+    /// can use spans to refer into it and give better error messages.
+    /// N.B: This does _not_ control whether implicit argument captures can be used.
+    pub is_source_literal: bool,
     /// Start position of the current line.
     cur_line_start: usize,
     /// Start and end byte offset of every line of the format string. Excludes
@@ -262,7 +265,7 @@ impl<'a> Iterator for Parser<'a> {
                     } else {
                         let arg = self.argument(lbrace_end);
                         if let Some(rbrace_pos) = self.must_consume('}') {
-                            if self.is_literal {
+                            if self.is_source_literal {
                                 let lbrace_byte_pos = self.to_span_index(pos);
                                 let rbrace_byte_pos = self.to_span_index(rbrace_pos);
 
@@ -302,7 +305,7 @@ impl<'a> Iterator for Parser<'a> {
                 _ => Some(String(self.string(pos))),
             }
         } else {
-            if self.is_literal {
+            if self.is_source_literal {
                 let span = self.span(self.cur_line_start, self.input.len());
                 if self.line_spans.last() != Some(&span) {
                     self.line_spans.push(span);
@@ -322,8 +325,8 @@ impl<'a> Parser<'a> {
         append_newline: bool,
         mode: ParseMode,
     ) -> Parser<'a> {
-        let input_string_kind = find_width_map_from_snippet(snippet, style);
-        let (width_map, is_literal) = match input_string_kind {
+        let input_string_kind = find_width_map_from_snippet(s, snippet, style);
+        let (width_map, is_source_literal) = match input_string_kind {
             InputStringKind::Literal { width_mappings } => (width_mappings, true),
             InputStringKind::NotALiteral => (Vec::new(), false),
         };
@@ -339,7 +342,7 @@ impl<'a> Parser<'a> {
             width_map,
             last_opening_brace: None,
             append_newline,
-            is_literal,
+            is_source_literal,
             cur_line_start: 0,
             line_spans: vec![],
         }
@@ -532,13 +535,13 @@ impl<'a> Parser<'a> {
                 '{' | '}' => {
                     return &self.input[start..pos];
                 }
-                '\n' if self.is_literal => {
+                '\n' if self.is_source_literal => {
                     self.line_spans.push(self.span(self.cur_line_start, pos));
                     self.cur_line_start = pos + 1;
                     self.cur.next();
                 }
                 _ => {
-                    if self.is_literal && pos == self.cur_line_start && c.is_whitespace() {
+                    if self.is_source_literal && pos == self.cur_line_start && c.is_whitespace() {
                         self.cur_line_start = pos + c.len_utf8();
                     }
                     self.cur.next();
@@ -890,6 +893,7 @@ impl<'a> Parser<'a> {
 /// written code (code snippet) and the `InternedString` that gets processed in the `Parser`
 /// in order to properly synthesise the intra-string `Span`s for error diagnostics.
 fn find_width_map_from_snippet(
+    input: &str,
     snippet: Option<string::String>,
     str_style: Option<usize>,
 ) -> InputStringKind {
@@ -902,8 +906,27 @@ fn find_width_map_from_snippet(
         return InputStringKind::Literal { width_mappings: Vec::new() };
     }
 
+    // Strip quotes.
     let snippet = &snippet[1..snippet.len() - 1];
 
+    // Macros like `println` add a newline at the end. That technically doesn't make them "literals" anymore, but it's fine
+    // since we will never need to point our spans there, so we lie about it here by ignoring it.
+    // Since there might actually be newlines in the source code, we need to normalize away all trailing newlines.
+    // If we only trimmed it off the input, `format!("\n")` would cause a mismatch as here we they actually match up.
+    // Alternatively, we could just count the trailing newlines and only trim one from the input if they don't match up.
+    let input_no_nl = input.trim_end_matches('\n');
+    let Some(unescaped) = unescape_string(snippet) else {
+        return InputStringKind::NotALiteral;
+    };
+
+    let unescaped_no_nl = unescaped.trim_end_matches('\n');
+
+    if unescaped_no_nl != input_no_nl {
+        // The source string that we're pointing at isn't our input, so spans pointing at it will be incorrect.
+        // This can for example happen with proc macros that respan generated literals.
+        return InputStringKind::NotALiteral;
+    }
+
     let mut s = snippet.char_indices();
     let mut width_mappings = vec![];
     while let Some((pos, c)) = s.next() {
@@ -986,6 +1009,19 @@ fn find_width_map_from_snippet(
     InputStringKind::Literal { width_mappings }
 }
 
+fn unescape_string(string: &str) -> Option<string::String> {
+    let mut buf = string::String::new();
+    let mut ok = true;
+    unescape::unescape_literal(string, unescape::Mode::Str, &mut |_, unescaped_char| {
+        match unescaped_char {
+            Ok(c) => buf.push(c),
+            Err(_) => ok = false,
+        }
+    });
+
+    ok.then_some(buf)
+}
+
 // Assert a reasonable size for `Piece`
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(Piece<'_>, 16);
diff --git a/compiler/rustc_passes/Cargo.toml b/compiler/rustc_passes/Cargo.toml
index faa9c493d..44f991f8c 100644
--- a/compiler/rustc_passes/Cargo.toml
+++ b/compiler/rustc_passes/Cargo.toml
@@ -22,3 +22,4 @@ rustc_span = { path = "../rustc_span" }
 rustc_lexer = { path = "../rustc_lexer" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_feature = { path = "../rustc_feature" }
+rustc_trait_selection = { path = "../rustc_trait_selection" }
diff --git a/compiler/rustc_passes/locales/en-US.ftl b/compiler/rustc_passes/messages.ftl
index 3fa78efc2..b354dca7c 100644
--- a/compiler/rustc_passes/locales/en-US.ftl
+++ b/compiler/rustc_passes/messages.ftl
@@ -148,9 +148,6 @@ passes_doc_test_unknown =
 passes_doc_test_takes_list =
     `#[doc(test(...)]` takes a list of attributes
 
-passes_doc_primitive =
-    `doc(primitive)` should never have been stable
-
 passes_doc_cfg_hide_takes_list =
     `#[doc(cfg_hide(...)]` takes a list of attributes
 
@@ -720,26 +717,7 @@ passes_ignored_derived_impls =
      *[other] traits {$trait_list}, but these are
     } intentionally ignored during dead code analysis
 
-passes_proc_macro_typeerror = mismatched {$kind} signature
-    .label = found {$found}, expected type `proc_macro::TokenStream`
-    .note = {$kind}s must have a signature of `{$expected_signature}`
-
-passes_proc_macro_diff_arg_count = mismatched {$kind} signature
-    .label = found unexpected {$count ->
-      [one] argument
-     *[other] arguments
-    }
-    .note = {$kind}s must have a signature of `{$expected_signature}`
-
-passes_proc_macro_missing_args = mismatched {$kind} signature
-    .label = {$kind} must have {$expected_input_count ->
-      [one] one argument
-     *[other] two arguments
-    } of type `proc_macro::TokenStream`
-
-passes_proc_macro_invalid_abi = proc macro functions may not be `extern "{$abi}"`
-
-passes_proc_macro_unsafe = proc macro functions may not be `unsafe`
+passes_proc_macro_bad_sig = {$kind} has incorrect signature
 
 passes_skipping_const_checks = skipping const checks
 
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 5ef3e13ef..80a93da2b 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -19,9 +19,10 @@ use rustc_hir::{
 use rustc_hir::{MethodKind, Target, Unsafety};
 use rustc_middle::hir::nested_filter;
 use rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault;
-use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_middle::traits::ObligationCause;
+use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{ParamEnv, TyCtxt};
+use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::lint::builtin::{
     CONFLICTING_REPR_HINTS, INVALID_DOC_ATTRIBUTES, INVALID_MACRO_EXPORT_ARGUMENTS,
     UNUSED_ATTRIBUTES,
@@ -30,6 +31,9 @@ use rustc_session::parse::feature_err;
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::spec::abi::Abi;
+use rustc_trait_selection::infer::{TyCtxtInferExt, ValuePairs};
+use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt;
+use rustc_trait_selection::traits::ObligationCtxt;
 use std::cell::Cell;
 use std::collections::hash_map::Entry;
 
@@ -1105,17 +1109,6 @@ impl CheckAttrVisitor<'_> {
                             }
                         }
 
-                        sym::primitive => {
-                            if !self.tcx.features().rustdoc_internals {
-                                self.tcx.emit_spanned_lint(
-                                    INVALID_DOC_ATTRIBUTES,
-                                    hir_id,
-                                    i_meta.span,
-                                    errors::DocPrimitive,
-                                );
-                            }
-                        }
-
                         _ => {
                             let path = rustc_ast_pretty::pprust::path_to_string(&i_meta.path);
                             if i_meta.has_name(sym::spotlight) {
@@ -1903,7 +1896,7 @@ impl CheckAttrVisitor<'_> {
         match target {
             Target::Fn => {
                 for attr in attrs {
-                    if self.tcx.sess.is_proc_macro_attr(attr) {
+                    if attr.is_proc_macro_attr() {
                         debug!("Is proc macro attr");
                         return true;
                     }
@@ -2188,100 +2181,99 @@ impl CheckAttrVisitor<'_> {
     ///
     /// If this best effort goes wrong, it will just emit a worse error later (see #102923)
     fn check_proc_macro(&self, hir_id: HirId, target: Target, kind: ProcMacroKind) {
-        let expected_input_count = match kind {
-            ProcMacroKind::Attribute => 2,
-            ProcMacroKind::Derive | ProcMacroKind::FunctionLike => 1,
-        };
-
-        let expected_signature = match kind {
-            ProcMacroKind::Attribute => "fn(TokenStream, TokenStream) -> TokenStream",
-            ProcMacroKind::Derive | ProcMacroKind::FunctionLike => "fn(TokenStream) -> TokenStream",
-        };
+        if target != Target::Fn {
+            return;
+        }
 
         let tcx = self.tcx;
-        if target == Target::Fn {
-            let Some(tokenstream) = tcx.get_diagnostic_item(sym::TokenStream) else {return};
-            let tokenstream = tcx.type_of(tokenstream).subst_identity();
-
-            let id = hir_id.expect_owner();
-            let hir_sig = tcx.hir().fn_sig_by_hir_id(hir_id).unwrap();
-
-            let sig =
-                tcx.liberate_late_bound_regions(id.to_def_id(), tcx.fn_sig(id).subst_identity());
-            let sig = tcx.normalize_erasing_regions(ParamEnv::empty(), sig);
-
-            // We don't currently require that the function signature is equal to
-            // `fn(TokenStream) -> TokenStream`, but instead monomorphizes to
-            // `fn(TokenStream) -> TokenStream` after some substitution of generic arguments.
-            //
-            // Properly checking this means pulling in additional `rustc` crates, so we don't.
-            let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsInfer };
-
-            if sig.abi != Abi::Rust {
-                tcx.sess.emit_err(errors::ProcMacroInvalidAbi {
-                    span: hir_sig.span,
-                    abi: sig.abi.name(),
-                });
-                self.abort.set(true);
-            }
+        let Some(token_stream_def_id) = tcx.get_diagnostic_item(sym::TokenStream) else { return; };
+        let Some(token_stream) = tcx.type_of(token_stream_def_id).no_bound_vars() else { return; };
 
-            if sig.unsafety == Unsafety::Unsafe {
-                tcx.sess.emit_err(errors::ProcMacroUnsafe { span: hir_sig.span });
-                self.abort.set(true);
-            }
+        let def_id = hir_id.expect_owner().def_id;
+        let param_env = ty::ParamEnv::empty();
 
-            let output = sig.output();
+        let infcx = tcx.infer_ctxt().build();
+        let ocx = ObligationCtxt::new(&infcx);
 
-            // Typecheck the output
-            if !drcx.types_may_unify(output, tokenstream) {
-                tcx.sess.emit_err(errors::ProcMacroTypeError {
-                    span: hir_sig.decl.output.span(),
-                    found: output,
-                    kind,
-                    expected_signature,
-                });
-                self.abort.set(true);
-            }
+        let span = tcx.def_span(def_id);
+        let fresh_substs = infcx.fresh_substs_for_item(span, def_id.to_def_id());
+        let sig = tcx.liberate_late_bound_regions(
+            def_id.to_def_id(),
+            tcx.fn_sig(def_id).subst(tcx, fresh_substs),
+        );
 
-            if sig.inputs().len() < expected_input_count {
-                tcx.sess.emit_err(errors::ProcMacroMissingArguments {
-                    expected_input_count,
-                    span: hir_sig.span,
-                    kind,
-                    expected_signature,
-                });
-                self.abort.set(true);
-            }
+        let mut cause = ObligationCause::misc(span, def_id);
+        let sig = ocx.normalize(&cause, param_env, sig);
 
-            // Check that the inputs are correct, if there are enough.
-            if sig.inputs().len() >= expected_input_count {
-                for (arg, input) in
-                    sig.inputs().iter().zip(hir_sig.decl.inputs).take(expected_input_count)
-                {
-                    if !drcx.types_may_unify(*arg, tokenstream) {
-                        tcx.sess.emit_err(errors::ProcMacroTypeError {
-                            span: input.span,
-                            found: *arg,
-                            kind,
-                            expected_signature,
-                        });
-                        self.abort.set(true);
+        // proc macro is not WF.
+        let errors = ocx.select_where_possible();
+        if !errors.is_empty() {
+            return;
+        }
+
+        let expected_sig = tcx.mk_fn_sig(
+            std::iter::repeat(token_stream).take(match kind {
+                ProcMacroKind::Attribute => 2,
+                ProcMacroKind::Derive | ProcMacroKind::FunctionLike => 1,
+            }),
+            token_stream,
+            false,
+            Unsafety::Normal,
+            Abi::Rust,
+        );
+
+        if let Err(terr) = ocx.eq(&cause, param_env, expected_sig, sig) {
+            let mut diag = tcx.sess.create_err(errors::ProcMacroBadSig { span, kind });
+
+            let hir_sig = tcx.hir().fn_sig_by_hir_id(hir_id);
+            if let Some(hir_sig) = hir_sig {
+                match terr {
+                    TypeError::ArgumentMutability(idx) | TypeError::ArgumentSorts(_, idx) => {
+                        if let Some(ty) = hir_sig.decl.inputs.get(idx) {
+                            diag.set_span(ty.span);
+                            cause.span = ty.span;
+                        } else if idx == hir_sig.decl.inputs.len() {
+                            let span = hir_sig.decl.output.span();
+                            diag.set_span(span);
+                            cause.span = span;
+                        }
                     }
+                    TypeError::ArgCount => {
+                        if let Some(ty) = hir_sig.decl.inputs.get(expected_sig.inputs().len()) {
+                            diag.set_span(ty.span);
+                            cause.span = ty.span;
+                        }
+                    }
+                    TypeError::UnsafetyMismatch(_) => {
+                        // FIXME: Would be nice if we had a span here..
+                    }
+                    TypeError::AbiMismatch(_) => {
+                        // FIXME: Would be nice if we had a span here..
+                    }
+                    TypeError::VariadicMismatch(_) => {
+                        // FIXME: Would be nice if we had a span here..
+                    }
+                    _ => {}
                 }
             }
 
-            // Check that there are not too many arguments
-            let body_id = tcx.hir().body_owned_by(id.def_id);
-            let excess = tcx.hir().body(body_id).params.get(expected_input_count..);
-            if let Some(excess @ [begin @ end] | excess @ [begin, .., end]) = excess {
-                tcx.sess.emit_err(errors::ProcMacroDiffArguments {
-                    span: begin.span.to(end.span),
-                    count: excess.len(),
-                    kind,
-                    expected_signature,
-                });
-                self.abort.set(true);
-            }
+            infcx.err_ctxt().note_type_err(
+                &mut diag,
+                &cause,
+                None,
+                Some(ValuePairs::Sigs(ExpectedFound { expected: expected_sig, found: sig })),
+                terr,
+                false,
+                false,
+            );
+            diag.emit();
+            self.abort.set(true);
+        }
+
+        let errors = ocx.select_all_or_error();
+        if !errors.is_empty() {
+            infcx.err_ctxt().report_fulfillment_errors(&errors);
+            self.abort.set(true);
         }
     }
 }
diff --git a/compiler/rustc_passes/src/check_const.rs b/compiler/rustc_passes/src/check_const.rs
index 526b829bf..30dd3e4d0 100644
--- a/compiler/rustc_passes/src/check_const.rs
+++ b/compiler/rustc_passes/src/check_const.rs
@@ -104,9 +104,7 @@ impl<'tcx> CheckConstVisitor<'tcx> {
 
             // If this crate is not using stability attributes, or this function is not claiming to be a
             // stable `const fn`, that is all that is required.
-            if !tcx.features().staged_api
-                || tcx.has_attr(def_id.to_def_id(), sym::rustc_const_unstable)
-            {
+            if !tcx.features().staged_api || tcx.has_attr(def_id, sym::rustc_const_unstable) {
                 return true;
             }
 
diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs
index e2f858a34..5cfe691df 100644
--- a/compiler/rustc_passes/src/dead.rs
+++ b/compiler/rustc_passes/src/dead.rs
@@ -2,8 +2,8 @@
 // closely. The idea is that all reachable symbols are live, codes called
 // from live codes are live, and everything else is dead.
 
+use hir::def_id::{LocalDefIdMap, LocalDefIdSet};
 use itertools::Itertools;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::MultiSpan;
 use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind, Res};
@@ -13,9 +13,10 @@ use rustc_hir::{Node, PatKind, TyKind};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::middle::privacy::Level;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self, DefIdTree, TyCtxt};
+use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::lint;
 use rustc_span::symbol::{sym, Symbol};
+use rustc_target::abi::FieldIdx;
 use std::mem;
 
 use crate::errors::{
@@ -45,17 +46,17 @@ struct MarkSymbolVisitor<'tcx> {
     worklist: Vec<LocalDefId>,
     tcx: TyCtxt<'tcx>,
     maybe_typeck_results: Option<&'tcx ty::TypeckResults<'tcx>>,
-    live_symbols: FxHashSet<LocalDefId>,
+    live_symbols: LocalDefIdSet,
     repr_has_repr_c: bool,
     repr_has_repr_simd: bool,
     in_pat: bool,
     ignore_variant_stack: Vec<DefId>,
     // maps from tuple struct constructors to tuple struct items
-    struct_constructors: FxHashMap<LocalDefId, LocalDefId>,
+    struct_constructors: LocalDefIdMap<LocalDefId>,
     // maps from ADTs to ignored derived traits (e.g. Debug and Clone)
     // and the span of their respective impl (i.e., part of the derive
     // macro)
-    ignored_derived_traits: FxHashMap<LocalDefId, Vec<(DefId, DefId)>>,
+    ignored_derived_traits: LocalDefIdMap<Vec<(DefId, DefId)>>,
 }
 
 impl<'tcx> MarkSymbolVisitor<'tcx> {
@@ -232,17 +233,23 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
             if let PatKind::Wild = pat.kind {
                 continue;
             }
-            self.insert_def_id(variant.fields[idx].did);
+            self.insert_def_id(variant.fields[FieldIdx::from_usize(idx)].did);
         }
     }
 
     fn mark_live_symbols(&mut self) {
-        let mut scanned = FxHashSet::default();
+        let mut scanned = LocalDefIdSet::default();
         while let Some(id) = self.worklist.pop() {
             if !scanned.insert(id) {
                 continue;
             }
 
+            // Avoid accessing the HIR for the synthesized associated type generated for RPITITs.
+            if self.tcx.opt_rpitit_info(id.to_def_id()).is_some() {
+                self.live_symbols.insert(id);
+                continue;
+            }
+
             // in the case of tuple struct constructors we want to check the item, not the generated
             // tuple struct constructor function
             let id = self.struct_constructors.get(&id).copied().unwrap_or(id);
@@ -463,9 +470,9 @@ impl<'tcx> Visitor<'tcx> for MarkSymbolVisitor<'tcx> {
 
 fn has_allow_dead_code_or_lang_attr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     fn has_lang_attr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
-        tcx.has_attr(def_id.to_def_id(), sym::lang)
+        tcx.has_attr(def_id, sym::lang)
             // Stable attribute for #[lang = "panic_impl"]
-            || tcx.has_attr(def_id.to_def_id(), sym::panic_handler)
+            || tcx.has_attr(def_id, sym::panic_handler)
     }
 
     fn has_allow_dead_code(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
@@ -506,7 +513,7 @@ fn has_allow_dead_code_or_lang_attr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool
 fn check_item<'tcx>(
     tcx: TyCtxt<'tcx>,
     worklist: &mut Vec<LocalDefId>,
-    struct_constructors: &mut FxHashMap<LocalDefId, LocalDefId>,
+    struct_constructors: &mut LocalDefIdMap<LocalDefId>,
     id: hir::ItemId,
 ) {
     let allow_dead_code = has_allow_dead_code_or_lang_attr(tcx, id.owner_id.def_id);
@@ -583,9 +590,7 @@ fn check_foreign_item(tcx: TyCtxt<'_>, worklist: &mut Vec<LocalDefId>, id: hir::
     }
 }
 
-fn create_and_seed_worklist(
-    tcx: TyCtxt<'_>,
-) -> (Vec<LocalDefId>, FxHashMap<LocalDefId, LocalDefId>) {
+fn create_and_seed_worklist(tcx: TyCtxt<'_>) -> (Vec<LocalDefId>, LocalDefIdMap<LocalDefId>) {
     let effective_visibilities = &tcx.effective_visibilities(());
     // see `MarkSymbolVisitor::struct_constructors`
     let mut struct_constructors = Default::default();
@@ -617,7 +622,7 @@ fn create_and_seed_worklist(
 fn live_symbols_and_ignored_derived_traits(
     tcx: TyCtxt<'_>,
     (): (),
-) -> (FxHashSet<LocalDefId>, FxHashMap<LocalDefId, Vec<(DefId, DefId)>>) {
+) -> (LocalDefIdSet, LocalDefIdMap<Vec<(DefId, DefId)>>) {
     let (worklist, struct_constructors) = create_and_seed_worklist(tcx);
     let mut symbol_visitor = MarkSymbolVisitor {
         worklist,
@@ -629,7 +634,7 @@ fn live_symbols_and_ignored_derived_traits(
         in_pat: false,
         ignore_variant_stack: vec![],
         struct_constructors,
-        ignored_derived_traits: FxHashMap::default(),
+        ignored_derived_traits: Default::default(),
     };
     symbol_visitor.mark_live_symbols();
     (symbol_visitor.live_symbols, symbol_visitor.ignored_derived_traits)
@@ -643,8 +648,8 @@ struct DeadVariant {
 
 struct DeadVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
-    live_symbols: &'tcx FxHashSet<LocalDefId>,
-    ignored_derived_traits: &'tcx FxHashMap<LocalDefId, Vec<(DefId, DefId)>>,
+    live_symbols: &'tcx LocalDefIdSet,
+    ignored_derived_traits: &'tcx LocalDefIdMap<Vec<(DefId, DefId)>>,
 }
 
 enum ShouldWarnAboutField {
@@ -695,6 +700,13 @@ impl<'tcx> DeadVisitor<'tcx> {
             .collect();
 
         let descr = tcx.def_descr(first_id.to_def_id());
+        // `impl` blocks are "batched" and (unlike other batching) might
+        // contain different kinds of associated items.
+        let descr = if dead_codes.iter().any(|did| tcx.def_descr(did.to_def_id()) != descr) {
+            "associated item"
+        } else {
+            descr
+        };
         let num = dead_codes.len();
         let multiple = num > 6;
         let name_list = names.into();
@@ -707,12 +719,12 @@ impl<'tcx> DeadVisitor<'tcx> {
 
         let parent_info = if let Some(parent_item) = parent_item {
             let parent_descr = tcx.def_descr(parent_item.to_def_id());
-            Some(ParentInfo {
-                num,
-                descr,
-                parent_descr,
-                span: tcx.def_ident_span(parent_item).unwrap(),
-            })
+            let span = if let DefKind::Impl { .. } = tcx.def_kind(parent_item) {
+                tcx.def_span(parent_item)
+            } else {
+                tcx.def_ident_span(parent_item).unwrap()
+            };
+            Some(ParentInfo { num, descr, parent_descr, span })
         } else {
             None
         };
@@ -795,16 +807,7 @@ impl<'tcx> DeadVisitor<'tcx> {
     }
 
     fn check_definition(&mut self, def_id: LocalDefId) {
-        if self.live_symbols.contains(&def_id) {
-            return;
-        }
-        if has_allow_dead_code_or_lang_attr(self.tcx, def_id) {
-            return;
-        }
-        let Some(name) = self.tcx.opt_item_name(def_id.to_def_id()) else {
-            return
-        };
-        if name.as_str().starts_with('_') {
+        if self.is_live_code(def_id) {
             return;
         }
         match self.tcx.def_kind(def_id) {
@@ -822,6 +825,18 @@ impl<'tcx> DeadVisitor<'tcx> {
             _ => {}
         }
     }
+
+    fn is_live_code(&self, def_id: LocalDefId) -> bool {
+        // if we cannot get a name for the item, then we just assume that it is
+        // live. I mean, we can't really emit a lint.
+        let Some(name) = self.tcx.opt_item_name(def_id.to_def_id()) else {
+            return true;
+        };
+
+        self.live_symbols.contains(&def_id)
+            || has_allow_dead_code_or_lang_attr(self.tcx, def_id)
+            || name.as_str().starts_with('_')
+    }
 }
 
 fn check_mod_deathness(tcx: TyCtxt<'_>, module: LocalDefId) {
@@ -831,6 +846,22 @@ fn check_mod_deathness(tcx: TyCtxt<'_>, module: LocalDefId) {
     let module_items = tcx.hir_module_items(module);
 
     for item in module_items.items() {
+        if let hir::ItemKind::Impl(impl_item) = tcx.hir().item(item).kind {
+            let mut dead_items = Vec::new();
+            for item in impl_item.items {
+                let did = item.id.owner_id.def_id;
+                if !visitor.is_live_code(did) {
+                    dead_items.push(did)
+                }
+            }
+            visitor.warn_multiple_dead_codes(
+                &dead_items,
+                "used",
+                Some(item.owner_id.def_id),
+                false,
+            );
+        }
+
         if !live_symbols.contains(&item.owner_id.def_id) {
             let parent = tcx.local_parent(item.owner_id.def_id);
             if parent != module && !live_symbols.contains(&parent) {
@@ -895,10 +926,6 @@ fn check_mod_deathness(tcx: TyCtxt<'_>, module: LocalDefId) {
         }
     }
 
-    for impl_item in module_items.impl_items() {
-        visitor.check_definition(impl_item.owner_id.def_id);
-    }
-
     for foreign_item in module_items.foreign_items() {
         visitor.check_definition(foreign_item.owner_id.def_id);
     }
diff --git a/compiler/rustc_passes/src/debugger_visualizer.rs b/compiler/rustc_passes/src/debugger_visualizer.rs
index aeacbaa67..9dd39a5c9 100644
--- a/compiler/rustc_passes/src/debugger_visualizer.rs
+++ b/compiler/rustc_passes/src/debugger_visualizer.rs
@@ -4,11 +4,9 @@ use hir::CRATE_HIR_ID;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_expand::base::resolve_path;
 use rustc_hir as hir;
-use rustc_hir::def_id::CrateNum;
 use rustc_hir::HirId;
-use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::TyCtxt;
-use rustc_span::def_id::LOCAL_CRATE;
+use rustc_middle::{query::LocalCrate, ty::query::Providers};
 use rustc_span::{sym, DebuggerVisualizerFile, DebuggerVisualizerType};
 
 use std::sync::Arc;
@@ -69,9 +67,7 @@ fn check_for_debugger_visualizer(
 }
 
 /// Traverses and collects the debugger visualizers for a specific crate.
-fn debugger_visualizers(tcx: TyCtxt<'_>, cnum: CrateNum) -> Vec<DebuggerVisualizerFile> {
-    assert_eq!(cnum, LOCAL_CRATE);
-
+fn debugger_visualizers(tcx: TyCtxt<'_>, _: LocalCrate) -> Vec<DebuggerVisualizerFile> {
     // Initialize the collector.
     let mut debugger_visualizers = FxHashSet::default();
 
diff --git a/compiler/rustc_passes/src/diagnostic_items.rs b/compiler/rustc_passes/src/diagnostic_items.rs
index 110eb210d..eb6ea673c 100644
--- a/compiler/rustc_passes/src/diagnostic_items.rs
+++ b/compiler/rustc_passes/src/diagnostic_items.rs
@@ -12,9 +12,10 @@
 use rustc_ast as ast;
 use rustc_hir::diagnostic_items::DiagnosticItems;
 use rustc_hir::OwnerId;
+use rustc_middle::query::LocalCrate;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::TyCtxt;
-use rustc_span::def_id::{CrateNum, DefId, LOCAL_CRATE};
+use rustc_span::def_id::{DefId, LOCAL_CRATE};
 use rustc_span::symbol::{sym, Symbol};
 
 use crate::errors::DuplicateDiagnosticItemInCrate;
@@ -62,9 +63,7 @@ fn extract(attrs: &[ast::Attribute]) -> Option<Symbol> {
 }
 
 /// Traverse and collect the diagnostic items in the current
-fn diagnostic_items(tcx: TyCtxt<'_>, cnum: CrateNum) -> DiagnosticItems {
-    assert_eq!(cnum, LOCAL_CRATE);
-
+fn diagnostic_items(tcx: TyCtxt<'_>, _: LocalCrate) -> DiagnosticItems {
     // Initialize the collector.
     let mut diagnostic_items = DiagnosticItems::default();
 
diff --git a/compiler/rustc_passes/src/entry.rs b/compiler/rustc_passes/src/entry.rs
index b327ba633..e3e4b73ef 100644
--- a/compiler/rustc_passes/src/entry.rs
+++ b/compiler/rustc_passes/src/entry.rs
@@ -1,10 +1,11 @@
+use rustc_ast::attr;
 use rustc_ast::entry::EntryPointType;
 use rustc_errors::error_code;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::{ItemId, Node, CRATE_HIR_ID};
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{DefIdTree, TyCtxt};
+use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{sigpipe, CrateType, EntryFnType};
 use rustc_session::parse::feature_err;
 use rustc_span::symbol::sym;
@@ -37,7 +38,7 @@ fn entry_fn(tcx: TyCtxt<'_>, (): ()) -> Option<(DefId, EntryFnType)> {
     }
 
     // If the user wants no main function at all, then stop here.
-    if tcx.sess.contains_name(&tcx.hir().attrs(CRATE_HIR_ID), sym::no_main) {
+    if attr::contains_name(&tcx.hir().attrs(CRATE_HIR_ID), sym::no_main) {
         return None;
     }
 
@@ -57,9 +58,9 @@ fn entry_fn(tcx: TyCtxt<'_>, (): ()) -> Option<(DefId, EntryFnType)> {
 // An equivalent optimization was not applied to the duplicated code in test_harness.rs.
 fn entry_point_type(ctxt: &EntryContext<'_>, id: ItemId, at_root: bool) -> EntryPointType {
     let attrs = ctxt.tcx.hir().attrs(id.hir_id());
-    if ctxt.tcx.sess.contains_name(attrs, sym::start) {
+    if attr::contains_name(attrs, sym::start) {
         EntryPointType::Start
-    } else if ctxt.tcx.sess.contains_name(attrs, sym::rustc_main) {
+    } else if attr::contains_name(attrs, sym::rustc_main) {
         EntryPointType::RustcMainAttr
     } else {
         if let Some(name) = ctxt.tcx.opt_item_name(id.owner_id.to_def_id())
@@ -78,7 +79,7 @@ fn entry_point_type(ctxt: &EntryContext<'_>, id: ItemId, at_root: bool) -> Entry
 
 fn attr_span_by_symbol(ctxt: &EntryContext<'_>, id: ItemId, sym: Symbol) -> Option<Span> {
     let attrs = ctxt.tcx.hir().attrs(id.hir_id());
-    ctxt.tcx.sess.find_by_name(attrs, sym).map(|attr| attr.span)
+    attr::find_by_name(attrs, sym).map(|attr| attr.span)
 }
 
 fn find_item(id: ItemId, ctxt: &mut EntryContext<'_>) {
@@ -186,7 +187,7 @@ fn sigpipe(tcx: TyCtxt<'_>, def_id: DefId) -> u8 {
 
 fn no_main_err(tcx: TyCtxt<'_>, visitor: &EntryContext<'_>) {
     let sp = tcx.def_span(CRATE_DEF_ID);
-    if *tcx.sess.parse_sess.reached_eof.borrow() {
+    if tcx.sess.parse_sess.reached_eof.load(rustc_data_structures::sync::Ordering::Relaxed) {
         // There's an unclosed brace that made the parser reach `Eof`, we shouldn't complain about
         // the missing `fn main()` then as it might have been hidden inside an unclosed block.
         tcx.sess.delay_span_bug(sp, "`main` not found, but expected unclosed brace error");
@@ -205,7 +206,7 @@ fn no_main_err(tcx: TyCtxt<'_>, visitor: &EntryContext<'_>) {
     // The file may be empty, which leads to the diagnostic machinery not emitting this
     // note. This is a relatively simple way to detect that case and emit a span-less
     // note instead.
-    let file_empty = !tcx.sess.source_map().lookup_line(sp.hi()).is_ok();
+    let file_empty = tcx.sess.source_map().lookup_line(sp.hi()).is_err();
 
     tcx.sess.emit_err(NoMainErr {
         sp,
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
index 9f1c0b5a0..139ba8c96 100644
--- a/compiler/rustc_passes/src/errors.rs
+++ b/compiler/rustc_passes/src/errors.rs
@@ -289,10 +289,6 @@ pub struct DocTestTakesList;
 pub struct DocCfgHideTakesList;
 
 #[derive(LintDiagnostic)]
-#[diag(passes_doc_primitive)]
-pub struct DocPrimitive;
-
-#[derive(LintDiagnostic)]
 #[diag(passes_doc_test_unknown_any)]
 pub struct DocTestUnknownAny {
     pub path: String,
@@ -1546,52 +1542,11 @@ pub struct ChangeFieldsToBeOfUnitType {
 }
 
 #[derive(Diagnostic)]
-#[diag(passes_proc_macro_typeerror)]
-#[note]
-pub(crate) struct ProcMacroTypeError<'tcx> {
-    #[primary_span]
-    #[label]
-    pub span: Span,
-    pub found: Ty<'tcx>,
-    pub kind: ProcMacroKind,
-    pub expected_signature: &'static str,
-}
-
-#[derive(Diagnostic)]
-#[diag(passes_proc_macro_diff_arg_count)]
-pub(crate) struct ProcMacroDiffArguments {
-    #[primary_span]
-    #[label]
-    pub span: Span,
-    pub count: usize,
-    pub kind: ProcMacroKind,
-    pub expected_signature: &'static str,
-}
-
-#[derive(Diagnostic)]
-#[diag(passes_proc_macro_missing_args)]
-pub(crate) struct ProcMacroMissingArguments {
+#[diag(passes_proc_macro_bad_sig)]
+pub(crate) struct ProcMacroBadSig {
     #[primary_span]
-    #[label]
     pub span: Span,
-    pub expected_input_count: usize,
     pub kind: ProcMacroKind,
-    pub expected_signature: &'static str,
-}
-
-#[derive(Diagnostic)]
-#[diag(passes_proc_macro_invalid_abi)]
-pub(crate) struct ProcMacroInvalidAbi {
-    #[primary_span]
-    pub span: Span,
-    pub abi: &'static str,
-}
-
-#[derive(Diagnostic)]
-#[diag(passes_proc_macro_unsafe)]
-pub(crate) struct ProcMacroUnsafe {
-    #[primary_span]
-    pub span: Span,
 }
 
 #[derive(Diagnostic)]
diff --git a/compiler/rustc_passes/src/hir_id_validator.rs b/compiler/rustc_passes/src/hir_id_validator.rs
index de0e50a65..3942a73be 100644
--- a/compiler/rustc_passes/src/hir_id_validator.rs
+++ b/compiler/rustc_passes/src/hir_id_validator.rs
@@ -5,11 +5,9 @@ use rustc_hir::intravisit;
 use rustc_hir::{HirId, ItemLocalId};
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_middle::hir::nested_filter;
-use rustc_middle::ty::{DefIdTree, TyCtxt};
+use rustc_middle::ty::TyCtxt;
 
 pub fn check_crate(tcx: TyCtxt<'_>) {
-    tcx.dep_graph.assert_ignored();
-
     if tcx.sess.opts.unstable_opts.hir_stats {
         crate::hir_stats::print_hir_stats(tcx);
     }
diff --git a/compiler/rustc_passes/src/hir_stats.rs b/compiler/rustc_passes/src/hir_stats.rs
index 5e2d2d3e5..47e032758 100644
--- a/compiler/rustc_passes/src/hir_stats.rs
+++ b/compiler/rustc_passes/src/hir_stats.rs
@@ -300,7 +300,7 @@ impl<'v> hir_visit::Visitor<'v> for StatCollector<'v> {
         record_variants!(
             (self, e, e.kind, Id::Node(e.hir_id), hir, Expr, ExprKind),
             [
-                Box, ConstBlock, Array, Call, MethodCall, Tup, Binary, Unary, Lit, Cast, Type,
+                ConstBlock, Array, Call, MethodCall, Tup, Binary, Unary, Lit, Cast, Type,
                 DropTemps, Let, If, Loop, Match, Closure, Block, Assign, AssignOp, Field, Index,
                 Path, AddrOf, Break, Continue, Ret, InlineAsm, Struct, Repeat, Yield, Err
             ]
@@ -565,7 +565,7 @@ impl<'v> ast_visit::Visitor<'v> for StatCollector<'v> {
         record_variants!(
             (self, e, e.kind, Id::None, ast, Expr, ExprKind),
             [
-                Box, Array, ConstBlock, Call, MethodCall, Tup, Binary, Unary, Lit, Cast, Type, Let,
+                Array, ConstBlock, Call, MethodCall, Tup, Binary, Unary, Lit, Cast, Type, Let,
                 If, While, ForLoop, Loop, Match, Closure, Block, Async, Await, TryBlock, Assign,
                 AssignOp, Field, Index, Range, Underscore, Path, AddrOf, Break, Continue, Ret,
                 InlineAsm, FormatArgs, MacCall, Struct, Repeat, Paren, Try, Yield, Yeet, IncludedBytes, Err
diff --git a/compiler/rustc_passes/src/layout_test.rs b/compiler/rustc_passes/src/layout_test.rs
index 047b9b525..5a1ae808e 100644
--- a/compiler/rustc_passes/src/layout_test.rs
+++ b/compiler/rustc_passes/src/layout_test.rs
@@ -18,7 +18,7 @@ pub fn test_layout(tcx: TyCtxt<'_>) {
                 tcx.def_kind(id.owner_id),
                 DefKind::TyAlias | DefKind::Enum | DefKind::Struct | DefKind::Union
             ) {
-                for attr in tcx.get_attrs(id.owner_id.to_def_id(), sym::rustc_layout) {
+                for attr in tcx.get_attrs(id.owner_id, sym::rustc_layout) {
                     dump_layout_of(tcx, id.owner_id.def_id, attr);
                 }
             }
diff --git a/compiler/rustc_passes/src/lib.rs b/compiler/rustc_passes/src/lib.rs
index 0cb842408..b7e07aff4 100644
--- a/compiler/rustc_passes/src/lib.rs
+++ b/compiler/rustc_passes/src/lib.rs
@@ -42,7 +42,7 @@ pub mod stability;
 mod upvars;
 mod weak_lang_items;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     check_attr::provide(providers);
diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs
index df5c8f53e..a8471ce3b 100644
--- a/compiler/rustc_passes/src/liveness.rs
+++ b/compiler/rustc_passes/src/liveness.rs
@@ -95,7 +95,7 @@ use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{Expr, HirId, HirIdMap, HirIdSet};
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self, DefIdTree, RootVariableMinCaptureList, Ty, TyCtxt};
+use rustc_middle::ty::{self, RootVariableMinCaptureList, Ty, TyCtxt};
 use rustc_session::lint;
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::{BytePos, Span};
@@ -146,7 +146,7 @@ fn check_liveness(tcx: TyCtxt<'_>, def_id: DefId) {
     // Don't run unused pass for #[derive()]
     let parent = tcx.local_parent(local_def_id);
     if let DefKind::Impl { .. } = tcx.def_kind(parent)
-        && tcx.has_attr(parent.to_def_id(), sym::automatically_derived)
+        && tcx.has_attr(parent, sym::automatically_derived)
     {
         return;
     }
@@ -473,7 +473,6 @@ impl<'tcx> Visitor<'tcx> for IrMaps<'tcx> {
             | hir::ExprKind::Struct(..)
             | hir::ExprKind::Repeat(..)
             | hir::ExprKind::InlineAsm(..)
-            | hir::ExprKind::Box(..)
             | hir::ExprKind::Type(..)
             | hir::ExprKind::Err(_)
             | hir::ExprKind::Path(hir::QPath::TypeRelative(..))
@@ -1059,8 +1058,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                 self.propagate_through_expr(&l, r_succ)
             }
 
-            hir::ExprKind::Box(ref e)
-            | hir::ExprKind::AddrOf(_, _, ref e)
+            hir::ExprKind::AddrOf(_, _, ref e)
             | hir::ExprKind::Cast(ref e, _)
             | hir::ExprKind::Type(ref e, _)
             | hir::ExprKind::DropTemps(ref e)
@@ -1425,7 +1423,6 @@ fn check_expr<'tcx>(this: &mut Liveness<'_, 'tcx>, expr: &'tcx Expr<'tcx>) {
         | hir::ExprKind::Closure { .. }
         | hir::ExprKind::Path(_)
         | hir::ExprKind::Yield(..)
-        | hir::ExprKind::Box(..)
         | hir::ExprKind::Type(..)
         | hir::ExprKind::Err(_) => {}
     }
diff --git a/compiler/rustc_passes/src/naked_functions.rs b/compiler/rustc_passes/src/naked_functions.rs
index c5b5cf7f5..c398467f0 100644
--- a/compiler/rustc_passes/src/naked_functions.rs
+++ b/compiler/rustc_passes/src/naked_functions.rs
@@ -30,7 +30,7 @@ fn check_mod_naked_functions(tcx: TyCtxt<'_>, module_def_id: LocalDefId) {
             continue;
         }
 
-        let naked = tcx.has_attr(def_id.to_def_id(), sym::naked);
+        let naked = tcx.has_attr(def_id, sym::naked);
         if !naked {
             continue;
         }
@@ -59,7 +59,7 @@ fn check_mod_naked_functions(tcx: TyCtxt<'_>, module_def_id: LocalDefId) {
 
 /// Check that the function isn't inlined.
 fn check_inline(tcx: TyCtxt<'_>, def_id: LocalDefId) {
-    let attrs = tcx.get_attrs(def_id.to_def_id(), sym::inline);
+    let attrs = tcx.get_attrs(def_id, sym::inline);
     for attr in attrs {
         tcx.sess.emit_err(CannotInlineNakedFunction { span: attr.span });
     }
@@ -179,8 +179,7 @@ enum ItemKind {
 impl<'tcx> CheckInlineAssembly<'tcx> {
     fn check_expr(&mut self, expr: &'tcx hir::Expr<'tcx>, span: Span) {
         match expr.kind {
-            ExprKind::Box(..)
-            | ExprKind::ConstBlock(..)
+            ExprKind::ConstBlock(..)
             | ExprKind::Array(..)
             | ExprKind::Call(..)
             | ExprKind::MethodCall(..)
diff --git a/compiler/rustc_passes/src/reachable.rs b/compiler/rustc_passes/src/reachable.rs
index 051100c56..a5f7b07fe 100644
--- a/compiler/rustc_passes/src/reachable.rs
+++ b/compiler/rustc_passes/src/reachable.rs
@@ -5,7 +5,7 @@
 // makes all other generics or inline functions that it references
 // reachable as well.
 
-use rustc_data_structures::fx::FxHashSet;
+use hir::def_id::LocalDefIdSet;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
@@ -14,7 +14,7 @@ use rustc_hir::Node;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
 use rustc_middle::middle::privacy::{self, Level};
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self, DefIdTree, TyCtxt};
+use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::config::CrateType;
 use rustc_target::spec::abi::Abi;
 
@@ -63,7 +63,7 @@ struct ReachableContext<'tcx> {
     tcx: TyCtxt<'tcx>,
     maybe_typeck_results: Option<&'tcx ty::TypeckResults<'tcx>>,
     // The set of items which must be exported in the linkage sense.
-    reachable_symbols: FxHashSet<LocalDefId>,
+    reachable_symbols: LocalDefIdSet,
     // A worklist of item IDs. Each item ID in this worklist will be inlined
     // and will be scanned for further references.
     // FIXME(eddyb) benchmark if this would be faster as a `VecDeque`.
@@ -175,7 +175,7 @@ impl<'tcx> ReachableContext<'tcx> {
 
     // Step 2: Mark all symbols that the symbols on the worklist touch.
     fn propagate(&mut self) {
-        let mut scanned = FxHashSet::default();
+        let mut scanned = LocalDefIdSet::default();
         while let Some(search_item) = self.worklist.pop() {
             if !scanned.insert(search_item) {
                 continue;
@@ -361,7 +361,7 @@ fn has_custom_linkage(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)
 }
 
-fn reachable_set(tcx: TyCtxt<'_>, (): ()) -> FxHashSet<LocalDefId> {
+fn reachable_set(tcx: TyCtxt<'_>, (): ()) -> LocalDefIdSet {
     let effective_visibilities = &tcx.effective_visibilities(());
 
     let any_library =
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index 16194a6f1..4a35c6794 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -159,7 +159,9 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
             return;
         }
 
-        let (stab, const_stab, body_stab) = attr::find_stability(&self.tcx.sess, attrs, item_sp);
+        let stab = attr::find_stability(&self.tcx.sess, attrs, item_sp);
+        let const_stab = attr::find_const_stability(&self.tcx.sess, attrs, item_sp);
+        let body_stab = attr::find_body_stability(&self.tcx.sess, attrs);
         let mut const_span = None;
 
         let const_stab = const_stab.map(|(const_stab, const_span_node)| {
@@ -265,6 +267,15 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
                 self.index.implications.insert(implied_by, feature);
             }
 
+            if let Some(ConstStability {
+                level: Unstable { implied_by: Some(implied_by), .. },
+                feature,
+                ..
+            }) = const_stab
+            {
+                self.index.implications.insert(implied_by, feature);
+            }
+
             self.index.stab_map.insert(def_id, stab);
             stab
         });
@@ -519,7 +530,7 @@ struct MissingStabilityAnnotations<'tcx> {
 impl<'tcx> MissingStabilityAnnotations<'tcx> {
     fn check_missing_stability(&self, def_id: LocalDefId, span: Span) {
         let stab = self.tcx.stability().local_stability(def_id);
-        if !self.tcx.sess.opts.test
+        if !self.tcx.sess.is_test_crate()
             && stab.is_none()
             && self.effective_visibilities.is_reachable(def_id)
         {
@@ -682,14 +693,10 @@ pub(crate) fn provide(providers: &mut Providers) {
         check_mod_unstable_api_usage,
         stability_index,
         stability_implications: |tcx, _| tcx.stability().implications.clone(),
-        lookup_stability: |tcx, id| tcx.stability().local_stability(id.expect_local()),
-        lookup_const_stability: |tcx, id| tcx.stability().local_const_stability(id.expect_local()),
-        lookup_default_body_stability: |tcx, id| {
-            tcx.stability().local_default_body_stability(id.expect_local())
-        },
-        lookup_deprecation_entry: |tcx, id| {
-            tcx.stability().local_deprecation_entry(id.expect_local())
-        },
+        lookup_stability: |tcx, id| tcx.stability().local_stability(id),
+        lookup_const_stability: |tcx, id| tcx.stability().local_const_stability(id),
+        lookup_default_body_stability: |tcx, id| tcx.stability().local_default_body_stability(id),
+        lookup_deprecation_entry: |tcx, id| tcx.stability().local_deprecation_entry(id),
         ..*providers
     };
 }
@@ -737,8 +744,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'tcx> {
                 let features = self.tcx.features();
                 if features.staged_api {
                     let attrs = self.tcx.hir().attrs(item.hir_id());
-                    let (stab, const_stab, _) =
-                        attr::find_stability(&self.tcx.sess, attrs, item.span);
+                    let stab = attr::find_stability(&self.tcx.sess, attrs, item.span);
+                    let const_stab = attr::find_const_stability(&self.tcx.sess, attrs, item.span);
 
                     // If this impl block has an #[unstable] attribute, give an
                     // error if all involved types and traits are stable, because
diff --git a/compiler/rustc_plugin_impl/locales/en-US.ftl b/compiler/rustc_plugin_impl/messages.ftl
index 8db32a42c..8db32a42c 100644
--- a/compiler/rustc_plugin_impl/locales/en-US.ftl
+++ b/compiler/rustc_plugin_impl/messages.ftl
diff --git a/compiler/rustc_plugin_impl/src/lib.rs b/compiler/rustc_plugin_impl/src/lib.rs
index 3f03eef9e..672189e22 100644
--- a/compiler/rustc_plugin_impl/src/lib.rs
+++ b/compiler/rustc_plugin_impl/src/lib.rs
@@ -18,7 +18,7 @@ use rustc_macros::fluent_messages;
 mod errors;
 pub mod load;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 /// Structure used to register plugins.
 ///
diff --git a/compiler/rustc_plugin_impl/src/load.rs b/compiler/rustc_plugin_impl/src/load.rs
index 8e75e969a..27e5cb9f0 100644
--- a/compiler/rustc_plugin_impl/src/load.rs
+++ b/compiler/rustc_plugin_impl/src/load.rs
@@ -3,7 +3,7 @@
 use crate::errors::{LoadPluginError, MalformedPluginAttribute};
 use crate::Registry;
 use libloading::Library;
-use rustc_ast::Crate;
+use rustc_ast::Attribute;
 use rustc_metadata::locator;
 use rustc_session::cstore::MetadataLoader;
 use rustc_session::Session;
@@ -20,11 +20,11 @@ type PluginRegistrarFn = fn(&mut Registry<'_>);
 pub fn load_plugins(
     sess: &Session,
     metadata_loader: &dyn MetadataLoader,
-    krate: &Crate,
+    attrs: &[Attribute],
 ) -> Vec<PluginRegistrarFn> {
     let mut plugins = Vec::new();
 
-    for attr in &krate.attrs {
+    for attr in attrs {
         if !attr.has_name(sym::plugin) {
             continue;
         }
diff --git a/compiler/rustc_privacy/locales/en-US.ftl b/compiler/rustc_privacy/messages.ftl
index a26d1b2b3..a26d1b2b3 100644
--- a/compiler/rustc_privacy/locales/en-US.ftl
+++ b/compiler/rustc_privacy/messages.ftl
diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs
index 50176c802..dcebfca08 100644
--- a/compiler/rustc_privacy/src/lib.rs
+++ b/compiler/rustc_privacy/src/lib.rs
@@ -29,7 +29,7 @@ use rustc_middle::middle::privacy::{EffectiveVisibilities, Level};
 use rustc_middle::span_bug;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::subst::InternalSubsts;
-use rustc_middle::ty::{self, Const, DefIdTree, GenericParamDefKind};
+use rustc_middle::ty::{self, Const, GenericParamDefKind};
 use rustc_middle::ty::{TraitRef, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor};
 use rustc_session::lint;
 use rustc_span::hygiene::Transparency;
@@ -46,7 +46,7 @@ use errors::{
     UnnamedItemIsPrivate,
 };
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 ////////////////////////////////////////////////////////////////////////////////
 /// Generic infrastructure used to implement specific visitors below.
@@ -132,7 +132,7 @@ where
                 projection.trait_ref_and_own_substs(tcx)
             } else {
                 // HACK(RPITIT): Remove this when RPITITs are lowered to regular assoc tys
-                let def_id = tcx.impl_trait_in_trait_parent(projection.def_id);
+                let def_id = tcx.impl_trait_in_trait_parent_fn(projection.def_id);
                 let trait_generics = tcx.generics_of(def_id);
                 (
                     tcx.mk_trait_ref(def_id, projection.substs.truncate_to(tcx, trait_generics)),
@@ -180,7 +180,7 @@ where
             | ty::PredicateKind::ConstEquate(_, _)
             | ty::PredicateKind::TypeWellFormedFromEnv(_)
             | ty::PredicateKind::Ambiguous
-            | ty::PredicateKind::AliasEq(_, _) => bug!("unexpected predicate: {:?}", predicate),
+            | ty::PredicateKind::AliasRelate(..) => bug!("unexpected predicate: {:?}", predicate),
         }
     }
 
@@ -515,16 +515,12 @@ impl<'tcx> EmbargoVisitor<'tcx> {
             let vis = self.tcx.local_visibility(item_id.owner_id.def_id);
             self.update_macro_reachable_def(item_id.owner_id.def_id, def_kind, vis, defining_mod);
         }
-        if let Some(exports) = self.tcx.module_reexports(module_def_id) {
-            for export in exports {
-                if export.vis.is_accessible_from(defining_mod, self.tcx) {
-                    if let Res::Def(def_kind, def_id) = export.res {
-                        if let Some(def_id) = def_id.as_local() {
-                            let vis = self.tcx.local_visibility(def_id);
-                            self.update_macro_reachable_def(def_id, def_kind, vis, defining_mod);
-                        }
-                    }
-                }
+        for export in self.tcx.module_children_reexports(module_def_id) {
+            if export.vis.is_accessible_from(defining_mod, self.tcx)
+                && let Res::Def(def_kind, def_id) = export.res
+                && let Some(def_id) = def_id.as_local() {
+                let vis = self.tcx.local_visibility(def_id);
+                self.update_macro_reachable_def(def_id, def_kind, vis, defining_mod);
             }
         }
     }
@@ -920,7 +916,7 @@ pub struct TestReachabilityVisitor<'tcx, 'a> {
 
 impl<'tcx, 'a> TestReachabilityVisitor<'tcx, 'a> {
     fn effective_visibility_diagnostic(&mut self, def_id: LocalDefId) {
-        if self.tcx.has_attr(def_id.to_def_id(), sym::rustc_effective_visibility) {
+        if self.tcx.has_attr(def_id, sym::rustc_effective_visibility) {
             let mut error_msg = String::new();
             let span = self.tcx.def_span(def_id.to_def_id());
             if let Some(effective_vis) = self.effective_visibilities.effective_vis(def_id) {
@@ -1082,7 +1078,7 @@ impl<'tcx> Visitor<'tcx> for NamePrivacyVisitor<'tcx> {
                 // If the expression uses FRU we need to make sure all the unmentioned fields
                 // are checked for privacy (RFC 736). Rather than computing the set of
                 // unmentioned fields, just check them all.
-                for (vf_index, variant_field) in variant.fields.iter().enumerate() {
+                for (vf_index, variant_field) in variant.fields.iter_enumerated() {
                     let field = fields
                         .iter()
                         .find(|f| self.typeck_results().field_index(f.hir_id) == vf_index);
@@ -2060,8 +2056,8 @@ pub fn provide(providers: &mut Providers) {
     };
 }
 
-fn visibility(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Visibility<DefId> {
-    local_visibility(tcx, def_id.expect_local()).to_def_id()
+fn visibility(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Visibility<DefId> {
+    local_visibility(tcx, def_id).to_def_id()
 }
 
 fn local_visibility(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Visibility {
@@ -2149,6 +2145,7 @@ fn effective_visibilities(tcx: TyCtxt<'_>, (): ()) -> &EffectiveVisibilities {
 
     let mut check_visitor =
         TestReachabilityVisitor { tcx, effective_visibilities: &visitor.effective_visibilities };
+    check_visitor.effective_visibility_diagnostic(CRATE_DEF_ID);
     tcx.hir().visit_all_item_likes_in_crate(&mut check_visitor);
 
     tcx.arena.alloc(visitor.effective_visibilities)
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
index 3e8a88c7e..b107a3f03 100644
--- a/compiler/rustc_query_impl/Cargo.toml
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -16,7 +16,7 @@ rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_query_system = { path = "../rustc_query_system" }
-rustc-rayon-core = { version = "0.4.0", optional = true }
+rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index d7708a3bc..7001a1eed 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -5,7 +5,6 @@
 #![feature(const_mut_refs)]
 #![feature(min_specialization)]
 #![feature(never_type)]
-#![feature(once_cell)]
 #![feature(rustc_attrs)]
 #![recursion_limit = "256"]
 #![allow(rustc::potential_query_instability)]
@@ -19,13 +18,16 @@ extern crate rustc_middle;
 
 use rustc_data_structures::sync::AtomicU64;
 use rustc_middle::arena::Arena;
-use rustc_middle::dep_graph::{self, DepKindStruct};
-use rustc_middle::query::Key;
+use rustc_middle::dep_graph::{self, DepKind, DepKindStruct};
+use rustc_middle::query::erase::{erase, restore, Erase};
+use rustc_middle::query::AsLocalKey;
 use rustc_middle::ty::query::{
     query_keys, query_provided, query_provided_to_value, query_storage, query_values,
 };
 use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
 use rustc_middle::ty::TyCtxt;
+use rustc_query_system::dep_graph::SerializedDepNodeIndex;
+use rustc_query_system::Value;
 use rustc_span::Span;
 
 #[macro_use]
@@ -43,6 +45,13 @@ pub use on_disk_cache::OnDiskCache;
 mod profiling_support;
 pub use self::profiling_support::alloc_self_profile_query_strings;
 
+/// This is implemented per query and restoring query values from their erased state.
+trait QueryConfigRestored<'tcx>: QueryConfig<QueryCtxt<'tcx>> + Default {
+    type RestoredValue;
+
+    fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue;
+}
+
 rustc_query_append! { define_queries! }
 
 impl<'tcx> Queries<'tcx> {
diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs
index 46e34462c..30477c7bd 100644
--- a/compiler/rustc_query_impl/src/on_disk_cache.rs
+++ b/compiler/rustc_query_impl/src/on_disk_cache.rs
@@ -13,7 +13,7 @@ use rustc_middle::mir::{self, interpret};
 use rustc_middle::ty::codec::{RefDecodable, TyDecoder, TyEncoder};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_query_system::dep_graph::DepContext;
-use rustc_query_system::query::{QueryCache, QueryContext, QuerySideEffects};
+use rustc_query_system::query::{QueryCache, QuerySideEffects};
 use rustc_serialize::{
     opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder},
     Decodable, Decoder, Encodable, Encoder,
@@ -388,6 +388,12 @@ impl<'sess> OnDiskCache<'sess> {
         debug_assert!(prev.is_none());
     }
 
+    /// Return whether the cached query result can be decoded.
+    pub fn loadable_from_disk(&self, dep_node_index: SerializedDepNodeIndex) -> bool {
+        self.query_result_index.contains_key(&dep_node_index)
+        // with_decoder is infallible, so we can stop here
+    }
+
     /// Returns the cached query result if there is something in the cache for
     /// the given `SerializedDepNodeIndex`; otherwise returns `None`.
     pub fn try_load_query_result<'tcx, T>(
@@ -398,7 +404,9 @@ impl<'sess> OnDiskCache<'sess> {
     where
         T: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
     {
-        self.load_indexed(tcx, dep_node_index, &self.query_result_index)
+        let opt_value = self.load_indexed(tcx, dep_node_index, &self.query_result_index);
+        debug_assert_eq!(opt_value.is_some(), self.loadable_from_disk(dep_node_index));
+        opt_value
     }
 
     /// Stores side effect emitted during computation of an anonymous query.
@@ -428,8 +436,8 @@ impl<'sess> OnDiskCache<'sess> {
         T: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
     {
         let pos = index.get(&dep_node_index).cloned()?;
-
-        self.with_decoder(tcx, pos, |decoder| Some(decode_tagged(decoder, dep_node_index)))
+        let value = self.with_decoder(tcx, pos, |decoder| decode_tagged(decoder, dep_node_index));
+        Some(value)
     }
 
     fn with_decoder<'a, 'tcx, T, F: for<'s> FnOnce(&mut CacheDecoder<'s, 'tcx>) -> T>(
@@ -1038,8 +1046,6 @@ impl<'a, 'tcx> Encoder for CacheEncoder<'a, 'tcx> {
         emit_i8(i8);
 
         emit_bool(bool);
-        emit_f64(f64);
-        emit_f32(f32);
         emit_char(char);
         emit_str(&str);
         emit_raw_bytes(&[u8]);
@@ -1056,24 +1062,24 @@ impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for [u8] {
     }
 }
 
-pub fn encode_query_results<'a, 'tcx, CTX, Q>(
-    tcx: CTX,
+pub(crate) fn encode_query_results<'a, 'tcx, Q>(
+    query: Q,
+    qcx: QueryCtxt<'tcx>,
     encoder: &mut CacheEncoder<'a, 'tcx>,
     query_result_index: &mut EncodedDepNodeIndex,
 ) where
-    CTX: QueryContext + 'tcx,
-    Q: super::QueryConfig<CTX>,
-    Q::Value: Encodable<CacheEncoder<'a, 'tcx>>,
+    Q: super::QueryConfigRestored<'tcx>,
+    Q::RestoredValue: Encodable<CacheEncoder<'a, 'tcx>>,
 {
-    let _timer = tcx
-        .dep_context()
+    let _timer = qcx
+        .tcx
         .profiler()
-        .verbose_generic_activity_with_arg("encode_query_results_for", std::any::type_name::<Q>());
+        .verbose_generic_activity_with_arg("encode_query_results_for", query.name());
 
-    assert!(Q::query_state(tcx).all_inactive());
-    let cache = Q::query_cache(tcx);
+    assert!(query.query_state(qcx).all_inactive());
+    let cache = query.query_cache(qcx);
     cache.iter(&mut |key, value, dep_node| {
-        if Q::cache_on_disk(*tcx.dep_context(), &key) {
+        if query.cache_on_disk(qcx.tcx, &key) {
             let dep_node = SerializedDepNodeIndex::new(dep_node.index());
 
             // Record position of the cache entry.
@@ -1081,7 +1087,7 @@ pub fn encode_query_results<'a, 'tcx, CTX, Q>(
 
             // Encode the type check tables with the `SerializedDepNodeIndex`
             // as tag.
-            encoder.encode_tagged(dep_node, value);
+            encoder.encode_tagged(dep_node, &Q::restore(*value));
         }
     });
 }
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index a8592bd70..afbead7d1 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -19,7 +19,7 @@ use rustc_query_system::ich::StableHashingContext;
 use rustc_query_system::query::{
     force_query, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects, QueryStackFrame,
 };
-use rustc_query_system::{LayoutOfDepth, QueryOverflow, Value};
+use rustc_query_system::{LayoutOfDepth, QueryOverflow};
 use rustc_serialize::Decodable;
 use rustc_session::Limit;
 use rustc_span::def_id::LOCAL_CRATE;
@@ -263,30 +263,30 @@ macro_rules! feedable {
 }
 
 macro_rules! hash_result {
-    ([]) => {{
-        Some(dep_graph::hash_result)
+    ([][$V:ty]) => {{
+        Some(|hcx, result| dep_graph::hash_result(hcx, &restore::<$V>(*result)))
     }};
-    ([(no_hash) $($rest:tt)*]) => {{
+    ([(no_hash) $($rest:tt)*][$V:ty]) => {{
         None
     }};
-    ([$other:tt $($modifiers:tt)*]) => {
-        hash_result!([$($modifiers)*])
+    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
+        hash_result!([$($modifiers)*][$($args)*])
     };
 }
 
-macro_rules! get_provider {
-    ([][$tcx:expr, $name:ident, $key:expr]) => {{
-        $tcx.queries.local_providers.$name
+macro_rules! call_provider {
+    ([][$qcx:expr, $name:ident, $key:expr]) => {{
+        ($qcx.queries.local_providers.$name)($qcx.tcx, $key)
     }};
-    ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
-        if $key.query_crate_is_local() {
-            $tcx.queries.local_providers.$name
+    ([(separate_provide_extern) $($rest:tt)*][$qcx:expr, $name:ident, $key:expr]) => {{
+        if let Some(key) = $key.as_local_key() {
+            ($qcx.queries.local_providers.$name)($qcx.tcx, key)
         } else {
-            $tcx.queries.extern_providers.$name
+            ($qcx.queries.extern_providers.$name)($qcx.tcx, $key)
         }
     }};
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
-        get_provider!([$($modifiers)*][$($args)*])
+        call_provider!([$($modifiers)*][$($args)*])
     };
 }
 
@@ -350,18 +350,25 @@ pub(crate) fn create_query_frame<
     QueryStackFrame::new(description, span, def_id, def_kind, kind, ty_adt_id, hash)
 }
 
-fn try_load_from_on_disk_cache<'tcx, Q>(tcx: TyCtxt<'tcx>, dep_node: DepNode)
+fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode)
 where
     Q: QueryConfig<QueryCtxt<'tcx>>,
-    Q::Key: DepNodeParams<TyCtxt<'tcx>>,
 {
     debug_assert!(tcx.dep_graph.is_green(&dep_node));
 
     let key = Q::Key::recover(tcx, &dep_node).unwrap_or_else(|| {
         panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)
     });
-    if Q::cache_on_disk(tcx, &key) {
-        let _ = Q::execute_query(tcx, key);
+    if query.cache_on_disk(tcx, &key) {
+        let _ = query.execute_query(tcx, key);
+    }
+}
+
+pub(crate) fn loadable_from_disk<'tcx>(tcx: QueryCtxt<'tcx>, id: SerializedDepNodeIndex) -> bool {
+    if let Some(cache) = tcx.on_disk_cache().as_ref() {
+        cache.loadable_from_disk(id)
+    } else {
+        false
     }
 }
 
@@ -375,11 +382,9 @@ where
     tcx.on_disk_cache().as_ref()?.try_load_query_result(*tcx, id)
 }
 
-fn force_from_dep_node<'tcx, Q>(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
+fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
 where
     Q: QueryConfig<QueryCtxt<'tcx>>,
-    Q::Key: DepNodeParams<TyCtxt<'tcx>>,
-    Q::Value: Value<TyCtxt<'tcx>, DepKind>,
 {
     // We must avoid ever having to call `force_from_dep_node()` for a
     // `DepNode::codegen_unit`:
@@ -403,7 +408,7 @@ where
         #[cfg(debug_assertions)]
         let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
         let tcx = QueryCtxt::from_tcx(tcx);
-        force_query::<Q, _, DepKind>(tcx, key, dep_node);
+        force_query(query, tcx, key, dep_node);
         true
     } else {
         false
@@ -412,7 +417,7 @@ where
 
 pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx>
 where
-    Q: QueryConfig<QueryCtxt<'tcx>>,
+    Q: QueryConfig<QueryCtxt<'tcx>> + Default,
     Q::Key: DepNodeParams<TyCtxt<'tcx>>,
 {
     let fingerprint_style = Q::Key::fingerprint_style();
@@ -431,8 +436,10 @@ where
         is_anon,
         is_eval_always,
         fingerprint_style,
-        force_from_dep_node: Some(force_from_dep_node::<Q>),
-        try_load_from_on_disk_cache: Some(try_load_from_on_disk_cache::<Q>),
+        force_from_dep_node: Some(|tcx, dep_node| force_from_dep_node(Q::default(), tcx, dep_node)),
+        try_load_from_on_disk_cache: Some(|tcx, dep_node| {
+            try_load_from_on_disk_cache(Q::default(), tcx, dep_node)
+        }),
     }
 }
 
@@ -462,54 +469,70 @@ macro_rules! define_queries {
         mod queries {
             use std::marker::PhantomData;
 
-            $(pub struct $name<'tcx> {
-                data: PhantomData<&'tcx ()>
-            })*
+            $(
+                #[derive(Copy, Clone, Default)]
+                pub struct $name<'tcx> {
+                    data: PhantomData<&'tcx ()>
+                }
+            )*
         }
 
         $(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
             type Key = query_keys::$name<'tcx>;
-            type Value = query_values::$name<'tcx>;
-            const NAME: &'static str = stringify!($name);
+            type Value = Erase<query_values::$name<'tcx>>;
+
+            #[inline(always)]
+            fn name(self) -> &'static str {
+                stringify!($name)
+            }
+
+            #[inline]
+            fn format_value(self) -> fn(&Self::Value) -> String {
+                |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value))
+            }
 
             #[inline]
-            fn cache_on_disk(tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
+            fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
                 ::rustc_middle::query::cached::$name(tcx, key)
             }
 
             type Cache = query_storage::$name<'tcx>;
 
             #[inline(always)]
-            fn query_state<'a>(tcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, crate::dep_graph::DepKind>
+            fn query_state<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, crate::dep_graph::DepKind>
                 where QueryCtxt<'tcx>: 'a
             {
                 &tcx.queries.$name
             }
 
             #[inline(always)]
-            fn query_cache<'a>(tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
+            fn query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
                 where 'tcx:'a
             {
                 &tcx.query_system.caches.$name
             }
 
-            fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
-                tcx.$name(key)
+            fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
+                erase(tcx.$name(key))
             }
 
             #[inline]
             #[allow(unused_variables)]
-            fn compute(qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
+            fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
                 query_provided_to_value::$name(
                     qcx.tcx,
-                    get_provider!([$($modifiers)*][qcx, $name, key])(qcx.tcx, key)
+                    call_provider!([$($modifiers)*][qcx, $name, key])
                 )
             }
 
             #[inline]
-            fn try_load_from_disk(_qcx: QueryCtxt<'tcx>, _key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self> {
+            fn try_load_from_disk(
+                self,
+                _qcx: QueryCtxt<'tcx>,
+                _key: &Self::Key
+            ) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self::Value> {
                 should_ever_cache_on_disk!([$($modifiers)*] {
-                    if Self::cache_on_disk(_qcx.tcx, _key) {
+                    if ::rustc_middle::query::cached::$name(_qcx.tcx, _key) {
                         Some(|qcx: QueryCtxt<'tcx>, dep_node| {
                             let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>(
                                 qcx,
@@ -525,15 +548,74 @@ macro_rules! define_queries {
                 })
             }
 
-            const ANON: bool = is_anon!([$($modifiers)*]);
-            const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]);
-            const DEPTH_LIMIT: bool = depth_limit!([$($modifiers)*]);
-            const FEEDABLE: bool = feedable!([$($modifiers)*]);
+            #[inline]
+            fn loadable_from_disk(
+                self,
+                _qcx: QueryCtxt<'tcx>,
+                _key: &Self::Key,
+                _index: SerializedDepNodeIndex,
+            ) -> bool {
+                should_ever_cache_on_disk!([$($modifiers)*] {
+                    self.cache_on_disk(_qcx.tcx, _key) &&
+                        $crate::plumbing::loadable_from_disk(_qcx, _index)
+                } {
+                    false
+                })
+            }
+
+            #[inline]
+            fn value_from_cycle_error(
+                self,
+                tcx: TyCtxt<'tcx>,
+                cycle: &[QueryInfo<DepKind>],
+            ) -> Self::Value {
+                let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle);
+                erase(result)
+            }
+
+            #[inline(always)]
+            fn anon(self) -> bool {
+                is_anon!([$($modifiers)*])
+            }
+
+            #[inline(always)]
+            fn eval_always(self) -> bool {
+                is_eval_always!([$($modifiers)*])
+            }
+
+            #[inline(always)]
+            fn depth_limit(self) -> bool {
+                depth_limit!([$($modifiers)*])
+            }
+
+            #[inline(always)]
+            fn feedable(self) -> bool {
+                feedable!([$($modifiers)*])
+            }
+
+            #[inline(always)]
+            fn dep_kind(self) -> rustc_middle::dep_graph::DepKind {
+                dep_graph::DepKind::$name
+            }
+
+            #[inline(always)]
+            fn handle_cycle_error(self) -> rustc_query_system::HandleCycleError {
+                handle_cycle_error!([$($modifiers)*])
+            }
+
+            #[inline(always)]
+            fn hash_result(self) -> rustc_query_system::query::HashResult<Self::Value> {
+                hash_result!([$($modifiers)*][query_values::$name<'tcx>])
+            }
+        })*
 
-            const DEP_KIND: rustc_middle::dep_graph::DepKind = dep_graph::DepKind::$name;
-            const HANDLE_CYCLE_ERROR: rustc_query_system::HandleCycleError = handle_cycle_error!([$($modifiers)*]);
+        $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> {
+            type RestoredValue = query_values::$name<'tcx>;
 
-            const HASH_RESULT: rustc_query_system::query::HashResult<QueryCtxt<'tcx>, Self> = hash_result!([$($modifiers)*]);
+            #[inline(always)]
+            fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
+                restore::<query_values::$name<'tcx>>(value)
+            }
         })*
 
         #[allow(nonstandard_style)]
@@ -649,8 +731,13 @@ macro_rules! define_queries {
                         string_cache,
                     )
                 },
-                encode_query_results: expand_if_cached!([$($modifiers)*], |tcx, encoder, query_result_index|
-                    $crate::on_disk_cache::encode_query_results::<_, super::queries::$name<'_>>(tcx, encoder, query_result_index)
+                encode_query_results: expand_if_cached!([$($modifiers)*], |qcx, encoder, query_result_index|
+                    $crate::on_disk_cache::encode_query_results::<super::queries::$name<'tcx>>(
+                        super::queries::$name::default(),
+                        qcx,
+                        encoder,
+                        query_result_index,
+                    )
                 ),
             }})*
         }
@@ -730,16 +817,22 @@ macro_rules! define_queries_struct {
 
             $($(#[$attr])*
             #[inline(always)]
-            #[tracing::instrument(level = "trace", skip(self, tcx), ret)]
+            #[tracing::instrument(level = "trace", skip(self, tcx))]
             fn $name(
                 &'tcx self,
                 tcx: TyCtxt<'tcx>,
                 span: Span,
-                key: <queries::$name<'tcx> as QueryConfig<QueryCtxt<'tcx>>>::Key,
+                key: query_keys::$name<'tcx>,
                 mode: QueryMode,
-            ) -> Option<query_values::$name<'tcx>> {
+            ) -> Option<Erase<query_values::$name<'tcx>>> {
                 let qcx = QueryCtxt { tcx, queries: self };
-                get_query::<queries::$name<'tcx>, _, rustc_middle::dep_graph::DepKind>(qcx, span, key, mode)
+                get_query(
+                    queries::$name::default(),
+                    qcx,
+                    span,
+                    key,
+                    mode
+                )
             })*
         }
     };
diff --git a/compiler/rustc_query_system/Cargo.toml b/compiler/rustc_query_system/Cargo.toml
index 7d8f75e25..12b4a1143 100644
--- a/compiler/rustc_query_system/Cargo.toml
+++ b/compiler/rustc_query_system/Cargo.toml
@@ -15,7 +15,7 @@ rustc_feature = { path = "../rustc_feature" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
-rustc-rayon-core = { version = "0.4.0", optional = true }
+rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
diff --git a/compiler/rustc_query_system/locales/en-US.ftl b/compiler/rustc_query_system/messages.ftl
index 870e82403..870e82403 100644
--- a/compiler/rustc_query_system/locales/en-US.ftl
+++ b/compiler/rustc_query_system/messages.ftl
diff --git a/compiler/rustc_query_system/src/cache.rs b/compiler/rustc_query_system/src/cache.rs
index 7cc885be2..6e862db0b 100644
--- a/compiler/rustc_query_system/src/cache.rs
+++ b/compiler/rustc_query_system/src/cache.rs
@@ -7,11 +7,16 @@ use rustc_data_structures::sync::Lock;
 
 use std::hash::Hash;
 
-#[derive(Clone)]
 pub struct Cache<Key, Value> {
     hashmap: Lock<FxHashMap<Key, WithDepNode<Value>>>,
 }
 
+impl<Key: Clone, Value: Clone> Clone for Cache<Key, Value> {
+    fn clone(&self) -> Self {
+        Self { hashmap: Lock::new(self.hashmap.borrow().clone()) }
+    }
+}
+
 impl<Key, Value> Default for Cache<Key, Value> {
     fn default() -> Self {
         Self { hashmap: Default::default() }
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 59e0c3597..a9a2e6dd0 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -6,7 +6,6 @@ use rustc_data_structures::sharded::{self, Sharded};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering};
-use rustc_data_structures::OnDrop;
 use rustc_index::vec::IndexVec;
 use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use smallvec::{smallvec, SmallVec};
@@ -54,6 +53,11 @@ impl From<DepNodeIndex> for QueryInvocationId {
     }
 }
 
+pub struct MarkFrame<'a> {
+    index: SerializedDepNodeIndex,
+    parent: Option<&'a MarkFrame<'a>>,
+}
+
 #[derive(PartialEq)]
 pub enum DepNodeColor {
     Red,
@@ -70,7 +74,7 @@ impl DepNodeColor {
     }
 }
 
-struct DepGraphData<K: DepKind> {
+pub struct DepGraphData<K: DepKind> {
     /// The new encoding of the dependency graph, optimized for red/green
     /// tracking. The `current` field is the dependency graph of only the
     /// current compilation session: We don't merge the previous dep-graph into
@@ -139,7 +143,7 @@ impl<K: DepKind> DepGraph<K> {
         assert_eq!(_green_node_index, DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE);
 
         // Instantiate a dependy-less red node only once for anonymous queries.
-        let (_red_node_index, _prev_and_index) = current.intern_node(
+        let (red_node_index, red_node_prev_index_and_color) = current.intern_node(
             profiler,
             &prev_graph,
             DepNode { kind: DepKind::RED, hash: Fingerprint::ZERO.into() },
@@ -147,8 +151,21 @@ impl<K: DepKind> DepGraph<K> {
             None,
             false,
         );
-        assert_eq!(_red_node_index, DepNodeIndex::FOREVER_RED_NODE);
-        assert!(matches!(_prev_and_index, None | Some((_, DepNodeColor::Red))));
+        assert_eq!(red_node_index, DepNodeIndex::FOREVER_RED_NODE);
+        match red_node_prev_index_and_color {
+            None => {
+                // This is expected when we have no previous compilation session.
+                assert!(prev_graph_node_count == 0);
+            }
+            Some((prev_red_node_index, DepNodeColor::Red)) => {
+                assert_eq!(prev_red_node_index.as_usize(), red_node_index.as_usize());
+                colors.insert(prev_red_node_index, DepNodeColor::Red);
+            }
+            Some((_, DepNodeColor::Green(_))) => {
+                // There must be a logic error somewhere if we hit this branch.
+                panic!("DepNodeIndex::FOREVER_RED_NODE evaluated to DepNodeColor::Green")
+            }
+        }
 
         DepGraph {
             data: Some(Lrc::new(DepGraphData {
@@ -168,6 +185,11 @@ impl<K: DepKind> DepGraph<K> {
         DepGraph { data: None, virtual_dep_node_index: Lrc::new(AtomicU32::new(0)) }
     }
 
+    #[inline]
+    pub fn data(&self) -> Option<&DepGraphData<K>> {
+        self.data.as_deref()
+    }
+
     /// Returns `true` if we are actually building the full dep-graph, and `false` otherwise.
     #[inline]
     pub fn is_fully_enabled(&self) -> bool {
@@ -252,6 +274,38 @@ impl<K: DepKind> DepGraph<K> {
         K::with_deps(TaskDepsRef::Forbid, op)
     }
 
+    #[inline(always)]
+    pub fn with_task<Ctxt: HasDepContext<DepKind = K>, A: Debug, R>(
+        &self,
+        key: DepNode<K>,
+        cx: Ctxt,
+        arg: A,
+        task: fn(Ctxt, A) -> R,
+        hash_result: Option<fn(&mut StableHashingContext<'_>, &R) -> Fingerprint>,
+    ) -> (R, DepNodeIndex) {
+        match self.data() {
+            Some(data) => data.with_task(key, cx, arg, task, hash_result),
+            None => (task(cx, arg), self.next_virtual_depnode_index()),
+        }
+    }
+
+    pub fn with_anon_task<Tcx: DepContext<DepKind = K>, OP, R>(
+        &self,
+        cx: Tcx,
+        dep_kind: K,
+        op: OP,
+    ) -> (R, DepNodeIndex)
+    where
+        OP: FnOnce() -> R,
+    {
+        match self.data() {
+            Some(data) => data.with_anon_task(cx, dep_kind, op),
+            None => (op(), self.next_virtual_depnode_index()),
+        }
+    }
+}
+
+impl<K: DepKind> DepGraphData<K> {
     /// Starts a new dep-graph task. Dep-graph tasks are specified
     /// using a free function (`task`) and **not** a closure -- this
     /// is intentional because we want to exercise tight control over
@@ -288,29 +342,6 @@ impl<K: DepKind> DepGraph<K> {
         task: fn(Ctxt, A) -> R,
         hash_result: Option<fn(&mut StableHashingContext<'_>, &R) -> Fingerprint>,
     ) -> (R, DepNodeIndex) {
-        if self.is_fully_enabled() {
-            self.with_task_impl(key, cx, arg, task, hash_result)
-        } else {
-            // Incremental compilation is turned off. We just execute the task
-            // without tracking. We still provide a dep-node index that uniquely
-            // identifies the task so that we have a cheap way of referring to
-            // the query for self-profiling.
-            (task(cx, arg), self.next_virtual_depnode_index())
-        }
-    }
-
-    #[inline(always)]
-    fn with_task_impl<Ctxt: HasDepContext<DepKind = K>, A: Debug, R>(
-        &self,
-        key: DepNode<K>,
-        cx: Ctxt,
-        arg: A,
-        task: fn(Ctxt, A) -> R,
-        hash_result: Option<fn(&mut StableHashingContext<'_>, &R) -> Fingerprint>,
-    ) -> (R, DepNodeIndex) {
-        // This function is only called when the graph is enabled.
-        let data = self.data.as_ref().unwrap();
-
         // If the following assertion triggers, it can have two reasons:
         // 1. Something is wrong with DepNode creation, either here or
         //    in `DepGraph::try_mark_green()`.
@@ -335,10 +366,8 @@ impl<K: DepKind> DepGraph<K> {
             }))
         };
 
-        let task_deps_ref = match &task_deps {
-            Some(deps) => TaskDepsRef::Allow(deps),
-            None => TaskDepsRef::Ignore,
-        };
+        let task_deps_ref =
+            task_deps.as_ref().map(TaskDepsRef::Allow).unwrap_or(TaskDepsRef::EvalAlways);
 
         let result = K::with_deps(task_deps_ref, || task(cx, arg));
         let edges = task_deps.map_or_else(|| smallvec![], |lock| lock.into_inner().reads);
@@ -351,9 +380,9 @@ impl<K: DepKind> DepGraph<K> {
         let print_status = cfg!(debug_assertions) && dcx.sess().opts.unstable_opts.dep_tasks;
 
         // Intern the new `DepNode`.
-        let (dep_node_index, prev_and_color) = data.current.intern_node(
+        let (dep_node_index, prev_and_color) = self.current.intern_node(
             dcx.profiler(),
-            &data.previous,
+            &self.previous,
             key,
             edges,
             current_fingerprint,
@@ -364,12 +393,12 @@ impl<K: DepKind> DepGraph<K> {
 
         if let Some((prev_index, color)) = prev_and_color {
             debug_assert!(
-                data.colors.get(prev_index).is_none(),
+                self.colors.get(prev_index).is_none(),
                 "DepGraph::with_task() - Duplicate DepNodeColor \
                             insertion for {key:?}"
             );
 
-            data.colors.insert(prev_index, color);
+            self.colors.insert(prev_index, color);
         }
 
         (result, dep_node_index)
@@ -388,63 +417,66 @@ impl<K: DepKind> DepGraph<K> {
     {
         debug_assert!(!cx.is_eval_always(dep_kind));
 
-        if let Some(ref data) = self.data {
-            let task_deps = Lock::new(TaskDeps::default());
-            let result = K::with_deps(TaskDepsRef::Allow(&task_deps), op);
-            let task_deps = task_deps.into_inner();
-            let task_deps = task_deps.reads;
-
-            let dep_node_index = match task_deps.len() {
-                0 => {
-                    // Because the dep-node id of anon nodes is computed from the sets of its
-                    // dependencies we already know what the ID of this dependency-less node is
-                    // going to be (i.e. equal to the precomputed
-                    // `SINGLETON_DEPENDENCYLESS_ANON_NODE`). As a consequence we can skip creating
-                    // a `StableHasher` and sending the node through interning.
-                    DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE
-                }
-                1 => {
-                    // When there is only one dependency, don't bother creating a node.
-                    task_deps[0]
-                }
-                _ => {
-                    // The dep node indices are hashed here instead of hashing the dep nodes of the
-                    // dependencies. These indices may refer to different nodes per session, but this isn't
-                    // a problem here because we that ensure the final dep node hash is per session only by
-                    // combining it with the per session random number `anon_id_seed`. This hash only need
-                    // to map the dependencies to a single value on a per session basis.
-                    let mut hasher = StableHasher::new();
-                    task_deps.hash(&mut hasher);
-
-                    let target_dep_node = DepNode {
-                        kind: dep_kind,
-                        // Fingerprint::combine() is faster than sending Fingerprint
-                        // through the StableHasher (at least as long as StableHasher
-                        // is so slow).
-                        hash: data.current.anon_id_seed.combine(hasher.finish()).into(),
-                    };
+        let task_deps = Lock::new(TaskDeps::default());
+        let result = K::with_deps(TaskDepsRef::Allow(&task_deps), op);
+        let task_deps = task_deps.into_inner();
+        let task_deps = task_deps.reads;
+
+        let dep_node_index = match task_deps.len() {
+            0 => {
+                // Because the dep-node id of anon nodes is computed from the sets of its
+                // dependencies we already know what the ID of this dependency-less node is
+                // going to be (i.e. equal to the precomputed
+                // `SINGLETON_DEPENDENCYLESS_ANON_NODE`). As a consequence we can skip creating
+                // a `StableHasher` and sending the node through interning.
+                DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE
+            }
+            1 => {
+                // When there is only one dependency, don't bother creating a node.
+                task_deps[0]
+            }
+            _ => {
+                // The dep node indices are hashed here instead of hashing the dep nodes of the
+                // dependencies. These indices may refer to different nodes per session, but this isn't
+                // a problem here because we that ensure the final dep node hash is per session only by
+                // combining it with the per session random number `anon_id_seed`. This hash only need
+                // to map the dependencies to a single value on a per session basis.
+                let mut hasher = StableHasher::new();
+                task_deps.hash(&mut hasher);
+
+                let target_dep_node = DepNode {
+                    kind: dep_kind,
+                    // Fingerprint::combine() is faster than sending Fingerprint
+                    // through the StableHasher (at least as long as StableHasher
+                    // is so slow).
+                    hash: self.current.anon_id_seed.combine(hasher.finish()).into(),
+                };
 
-                    data.current.intern_new_node(
-                        cx.profiler(),
-                        target_dep_node,
-                        task_deps,
-                        Fingerprint::ZERO,
-                    )
-                }
-            };
+                self.current.intern_new_node(
+                    cx.profiler(),
+                    target_dep_node,
+                    task_deps,
+                    Fingerprint::ZERO,
+                )
+            }
+        };
 
-            (result, dep_node_index)
-        } else {
-            (op(), self.next_virtual_depnode_index())
-        }
+        (result, dep_node_index)
     }
+}
 
+impl<K: DepKind> DepGraph<K> {
     #[inline]
     pub fn read_index(&self, dep_node_index: DepNodeIndex) {
         if let Some(ref data) = self.data {
             K::read_deps(|task_deps| {
                 let mut task_deps = match task_deps {
                     TaskDepsRef::Allow(deps) => deps.lock(),
+                    TaskDepsRef::EvalAlways => {
+                        // We don't need to record dependencies of eval_always
+                        // queries. They are re-evaluated unconditionally anyway.
+                        return;
+                    }
                     TaskDepsRef::Ignore => return,
                     TaskDepsRef::Forbid => {
                         panic!("Illegal read of: {dep_node_index:?}")
@@ -519,22 +551,38 @@ impl<K: DepKind> DepGraph<K> {
             // value to an existing node.
             //
             // For sanity, we still check that the loaded stable hash and the new one match.
-            if let Some(dep_node_index) = self.dep_node_index_of_opt(&node) {
-                let _current_fingerprint =
-                    crate::query::incremental_verify_ich(cx, result, &node, hash_result);
+            if let Some(prev_index) = data.previous.node_to_index_opt(&node) {
+                let dep_node_index = data.current.prev_index_to_index.lock()[prev_index];
+                if let Some(dep_node_index) = dep_node_index {
+                    crate::query::incremental_verify_ich(
+                        cx,
+                        data,
+                        result,
+                        prev_index,
+                        hash_result,
+                        |value| format!("{:?}", value),
+                    );
 
-                #[cfg(debug_assertions)]
-                if hash_result.is_some() {
-                    data.current.record_edge(dep_node_index, node, _current_fingerprint);
-                }
+                    #[cfg(debug_assertions)]
+                    if hash_result.is_some() {
+                        data.current.record_edge(
+                            dep_node_index,
+                            node,
+                            data.prev_fingerprint_of(prev_index),
+                        );
+                    }
 
-                return dep_node_index;
+                    return dep_node_index;
+                }
             }
 
             let mut edges = SmallVec::new();
             K::read_deps(|task_deps| match task_deps {
                 TaskDepsRef::Allow(deps) => edges.extend(deps.lock().reads.iter().copied()),
-                TaskDepsRef::Ignore => {} // During HIR lowering, we have no dependencies.
+                TaskDepsRef::EvalAlways => {
+                    edges.push(DepNodeIndex::FOREVER_RED_NODE);
+                }
+                TaskDepsRef::Ignore => {}
                 TaskDepsRef::Forbid => {
                     panic!("Cannot summarize when dependencies are not recorded.")
                 }
@@ -577,32 +625,63 @@ impl<K: DepKind> DepGraph<K> {
             self.next_virtual_depnode_index()
         }
     }
+}
 
+impl<K: DepKind> DepGraphData<K> {
     #[inline]
-    pub fn dep_node_index_of(&self, dep_node: &DepNode<K>) -> DepNodeIndex {
-        self.dep_node_index_of_opt(dep_node).unwrap()
+    pub fn dep_node_index_of_opt(&self, dep_node: &DepNode<K>) -> Option<DepNodeIndex> {
+        if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) {
+            self.current.prev_index_to_index.lock()[prev_index]
+        } else {
+            self.current
+                .new_node_to_index
+                .get_shard_by_value(dep_node)
+                .lock()
+                .get(dep_node)
+                .copied()
+        }
     }
 
     #[inline]
-    pub fn dep_node_index_of_opt(&self, dep_node: &DepNode<K>) -> Option<DepNodeIndex> {
-        let data = self.data.as_ref().unwrap();
-        let current = &data.current;
+    pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
+        self.dep_node_index_of_opt(dep_node).is_some()
+    }
 
-        if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) {
-            current.prev_index_to_index.lock()[prev_index]
+    fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
+        if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) {
+            self.colors.get(prev_index)
         } else {
-            current.new_node_to_index.get_shard_by_value(dep_node).lock().get(dep_node).copied()
+            // This is a node that did not exist in the previous compilation session.
+            None
         }
     }
 
+    /// Returns true if the given node has been marked as green during the
+    /// current compilation session. Used in various assertions
     #[inline]
-    pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
-        self.data.is_some() && self.dep_node_index_of_opt(dep_node).is_some()
+    pub fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
+        self.colors.get(prev_index).map_or(false, |c| c.is_green())
+    }
+
+    #[inline]
+    pub fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
+        self.previous.fingerprint_by_index(prev_index)
+    }
+
+    #[inline]
+    pub fn prev_node_of(&self, prev_index: SerializedDepNodeIndex) -> DepNode<K> {
+        self.previous.index_to_node(prev_index)
+    }
+
+    pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) {
+        self.debug_loaded_from_disk.lock().insert(dep_node);
     }
+}
 
+impl<K: DepKind> DepGraph<K> {
     #[inline]
-    pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
-        self.data.as_ref().unwrap().previous.fingerprint_of(dep_node)
+    pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
+        self.data.as_ref().map_or(false, |data| data.dep_node_exists(dep_node))
     }
 
     /// Checks whether a previous work product exists for `v` and, if
@@ -617,10 +696,6 @@ impl<K: DepKind> DepGraph<K> {
         &self.data.as_ref().unwrap().previous_work_products
     }
 
-    pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) {
-        self.data.as_ref().unwrap().debug_loaded_from_disk.lock().insert(dep_node);
-    }
-
     pub fn debug_was_loaded_from_disk(&self, dep_node: DepNode<K>) -> bool {
         self.data.as_ref().unwrap().debug_loaded_from_disk.lock().contains(&dep_node)
     }
@@ -645,17 +720,22 @@ impl<K: DepKind> DepGraph<K> {
 
     fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
         if let Some(ref data) = self.data {
-            if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) {
-                return data.colors.get(prev_index);
-            } else {
-                // This is a node that did not exist in the previous compilation session.
-                return None;
-            }
+            return data.node_color(dep_node);
         }
 
         None
     }
 
+    pub fn try_mark_green<Qcx: QueryContext<DepKind = K>>(
+        &self,
+        qcx: Qcx,
+        dep_node: &DepNode<K>,
+    ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> {
+        self.data().and_then(|data| data.try_mark_green(qcx, dep_node))
+    }
+}
+
+impl<K: DepKind> DepGraphData<K> {
     /// Try to mark a node index for the node dep_node.
     ///
     /// A node will have an index, when it's already been marked green, or when we can mark it
@@ -668,43 +748,33 @@ impl<K: DepKind> DepGraph<K> {
     ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> {
         debug_assert!(!qcx.dep_context().is_eval_always(dep_node.kind));
 
-        // Return None if the dep graph is disabled
-        let data = self.data.as_ref()?;
-
         // Return None if the dep node didn't exist in the previous session
-        let prev_index = data.previous.node_to_index_opt(dep_node)?;
+        let prev_index = self.previous.node_to_index_opt(dep_node)?;
 
-        match data.colors.get(prev_index) {
-            Some(DepNodeColor::Green(dep_node_index)) => return Some((prev_index, dep_node_index)),
-            Some(DepNodeColor::Red) => return None,
-            None => {}
+        match self.colors.get(prev_index) {
+            Some(DepNodeColor::Green(dep_node_index)) => Some((prev_index, dep_node_index)),
+            Some(DepNodeColor::Red) => None,
+            None => {
+                // This DepNode and the corresponding query invocation existed
+                // in the previous compilation session too, so we can try to
+                // mark it as green by recursively marking all of its
+                // dependencies green.
+                self.try_mark_previous_green(qcx, prev_index, &dep_node, None)
+                    .map(|dep_node_index| (prev_index, dep_node_index))
+            }
         }
-
-        let backtrace = backtrace_printer(qcx.dep_context().sess(), data, prev_index);
-
-        // This DepNode and the corresponding query invocation existed
-        // in the previous compilation session too, so we can try to
-        // mark it as green by recursively marking all of its
-        // dependencies green.
-        let ret = self
-            .try_mark_previous_green(qcx, data, prev_index, &dep_node)
-            .map(|dep_node_index| (prev_index, dep_node_index));
-
-        // We succeeded, no backtrace.
-        backtrace.disable();
-        return ret;
     }
 
-    #[instrument(skip(self, qcx, data, parent_dep_node_index), level = "debug")]
+    #[instrument(skip(self, qcx, parent_dep_node_index, frame), level = "debug")]
     fn try_mark_parent_green<Qcx: QueryContext<DepKind = K>>(
         &self,
         qcx: Qcx,
-        data: &DepGraphData<K>,
         parent_dep_node_index: SerializedDepNodeIndex,
         dep_node: &DepNode<K>,
+        frame: Option<&MarkFrame<'_>>,
     ) -> Option<()> {
-        let dep_dep_node_color = data.colors.get(parent_dep_node_index);
-        let dep_dep_node = &data.previous.index_to_node(parent_dep_node_index);
+        let dep_dep_node_color = self.colors.get(parent_dep_node_index);
+        let dep_dep_node = &self.previous.index_to_node(parent_dep_node_index);
 
         match dep_dep_node_color {
             Some(DepNodeColor::Green(_)) => {
@@ -734,7 +804,7 @@ impl<K: DepKind> DepGraph<K> {
             );
 
             let node_index =
-                self.try_mark_previous_green(qcx, data, parent_dep_node_index, dep_dep_node);
+                self.try_mark_previous_green(qcx, parent_dep_node_index, dep_dep_node, frame);
 
             if node_index.is_some() {
                 debug!("managed to MARK dependency {dep_dep_node:?} as green",);
@@ -744,13 +814,13 @@ impl<K: DepKind> DepGraph<K> {
 
         // We failed to mark it green, so we try to force the query.
         debug!("trying to force dependency {dep_dep_node:?}");
-        if !qcx.dep_context().try_force_from_dep_node(*dep_dep_node) {
+        if !qcx.dep_context().try_force_from_dep_node(*dep_dep_node, frame) {
             // The DepNode could not be forced.
             debug!("dependency {dep_dep_node:?} could not be forced");
             return None;
         }
 
-        let dep_dep_node_color = data.colors.get(parent_dep_node_index);
+        let dep_dep_node_color = self.colors.get(parent_dep_node_index);
 
         match dep_dep_node_color {
             Some(DepNodeColor::Green(_)) => {
@@ -783,32 +853,31 @@ impl<K: DepKind> DepGraph<K> {
     }
 
     /// Try to mark a dep-node which existed in the previous compilation session as green.
-    #[instrument(skip(self, qcx, data, prev_dep_node_index), level = "debug")]
+    #[instrument(skip(self, qcx, prev_dep_node_index, frame), level = "debug")]
     fn try_mark_previous_green<Qcx: QueryContext<DepKind = K>>(
         &self,
         qcx: Qcx,
-        data: &DepGraphData<K>,
         prev_dep_node_index: SerializedDepNodeIndex,
         dep_node: &DepNode<K>,
+        frame: Option<&MarkFrame<'_>>,
     ) -> Option<DepNodeIndex> {
+        let frame = MarkFrame { index: prev_dep_node_index, parent: frame };
+
         #[cfg(not(parallel_compiler))]
         {
             debug_assert!(!self.dep_node_exists(dep_node));
-            debug_assert!(data.colors.get(prev_dep_node_index).is_none());
+            debug_assert!(self.colors.get(prev_dep_node_index).is_none());
         }
 
         // We never try to mark eval_always nodes as green
         debug_assert!(!qcx.dep_context().is_eval_always(dep_node.kind));
 
-        debug_assert_eq!(data.previous.index_to_node(prev_dep_node_index), *dep_node);
+        debug_assert_eq!(self.previous.index_to_node(prev_dep_node_index), *dep_node);
 
-        let prev_deps = data.previous.edge_targets_from(prev_dep_node_index);
+        let prev_deps = self.previous.edge_targets_from(prev_dep_node_index);
 
         for &dep_dep_node_index in prev_deps {
-            let backtrace = backtrace_printer(qcx.dep_context().sess(), data, dep_dep_node_index);
-            let success = self.try_mark_parent_green(qcx, data, dep_dep_node_index, dep_node);
-            backtrace.disable();
-            success?;
+            self.try_mark_parent_green(qcx, dep_dep_node_index, dep_node, Some(&frame))?;
         }
 
         // If we got here without hitting a `return` that means that all
@@ -819,9 +888,9 @@ impl<K: DepKind> DepGraph<K> {
 
         // We allocating an entry for the node in the current dependency graph and
         // adding all the appropriate edges imported from the previous graph
-        let dep_node_index = data.current.promote_node_and_deps_to_current(
+        let dep_node_index = self.current.promote_node_and_deps_to_current(
             qcx.dep_context().profiler(),
-            &data.previous,
+            &self.previous,
             prev_dep_node_index,
         );
 
@@ -833,20 +902,20 @@ impl<K: DepKind> DepGraph<K> {
 
         #[cfg(not(parallel_compiler))]
         debug_assert!(
-            data.colors.get(prev_dep_node_index).is_none(),
+            self.colors.get(prev_dep_node_index).is_none(),
             "DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \
                       insertion for {dep_node:?}"
         );
 
         if !side_effects.is_empty() {
-            self.with_query_deserialization(|| {
-                self.emit_side_effects(qcx, data, dep_node_index, side_effects)
+            qcx.dep_context().dep_graph().with_query_deserialization(|| {
+                self.emit_side_effects(qcx, dep_node_index, side_effects)
             });
         }
 
         // ... and finally storing a "Green" entry in the color map.
         // Multiple threads can all write the same color here
-        data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index));
+        self.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index));
 
         debug!("successfully marked {dep_node:?} as green");
         Some(dep_node_index)
@@ -859,11 +928,10 @@ impl<K: DepKind> DepGraph<K> {
     fn emit_side_effects<Qcx: QueryContext<DepKind = K>>(
         &self,
         qcx: Qcx,
-        data: &DepGraphData<K>,
         dep_node_index: DepNodeIndex,
         side_effects: QuerySideEffects,
     ) {
-        let mut processed = data.processed_side_effects.lock();
+        let mut processed = self.processed_side_effects.lock();
 
         if processed.insert(dep_node_index) {
             // We were the first to insert the node in the set so this thread
@@ -879,7 +947,9 @@ impl<K: DepKind> DepGraph<K> {
             }
         }
     }
+}
 
+impl<K: DepKind> DepGraph<K> {
     /// Returns true if the given node has been marked as red during the
     /// current compilation session. Used in various assertions
     pub fn is_red(&self, dep_node: &DepNode<K>) -> bool {
@@ -937,6 +1007,7 @@ impl<K: DepKind> DepGraph<K> {
     }
 
     pub(crate) fn next_virtual_depnode_index(&self) -> DepNodeIndex {
+        debug_assert!(self.data.is_none());
         let index = self.virtual_dep_node_index.fetch_add(1, Relaxed);
         DepNodeIndex::from_u32(index)
     }
@@ -1020,7 +1091,7 @@ pub(super) struct CurrentDepGraph<K: DepKind> {
     /// This is used to verify that fingerprints do not change between the creation of a node
     /// and its recomputation.
     #[cfg(debug_assertions)]
-    fingerprints: Lock<FxHashMap<DepNode<K>, Fingerprint>>,
+    fingerprints: Lock<IndexVec<DepNodeIndex, Option<Fingerprint>>>,
 
     /// Used to trap when a specific edge is added to the graph.
     /// This is used for debug purposes and is only active with `debug_assertions`.
@@ -1106,7 +1177,7 @@ impl<K: DepKind> CurrentDepGraph<K> {
             #[cfg(debug_assertions)]
             forbidden_edge,
             #[cfg(debug_assertions)]
-            fingerprints: Lock::new(Default::default()),
+            fingerprints: Lock::new(IndexVec::from_elem_n(None, new_node_count_estimate)),
             total_read_count: AtomicU64::new(0),
             total_duplicate_read_count: AtomicU64::new(0),
             node_intern_event_id,
@@ -1118,14 +1189,8 @@ impl<K: DepKind> CurrentDepGraph<K> {
         if let Some(forbidden_edge) = &self.forbidden_edge {
             forbidden_edge.index_to_node.lock().insert(dep_node_index, key);
         }
-        match self.fingerprints.lock().entry(key) {
-            Entry::Vacant(v) => {
-                v.insert(fingerprint);
-            }
-            Entry::Occupied(o) => {
-                assert_eq!(*o.get(), fingerprint, "Unstable fingerprints for {:?}", key);
-            }
-        }
+        let previous = *self.fingerprints.lock().get_or_insert_with(dep_node_index, || fingerprint);
+        assert_eq!(previous, fingerprint, "Unstable fingerprints for {:?}", key);
     }
 
     /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it.
@@ -1310,10 +1375,13 @@ pub enum TaskDepsRef<'a, K: DepKind> {
     /// `TaskDeps`. This is used when executing a 'normal' query
     /// (no `eval_always` modifier)
     Allow(&'a Lock<TaskDeps<K>>),
-    /// New dependencies are ignored. This is used when
-    /// executing an `eval_always` query, since there's no
+    /// This is used when executing an `eval_always` query. We don't
     /// need to track dependencies for a query that's always
-    /// re-executed. This is also used for `dep_graph.with_ignore`
+    /// re-executed -- but we need to know that this is an `eval_always`
+    /// query in order to emit dependencies to `DepNodeIndex::FOREVER_RED_NODE`
+    /// when directly feeding other queries.
+    EvalAlways,
+    /// New dependencies are ignored. This is also used for `dep_graph.with_ignore`.
     Ignore,
     /// Any attempt to add new dependencies will cause a panic.
     /// This is used when decoding a query result from disk,
@@ -1381,25 +1449,25 @@ impl DepNodeColorMap {
     }
 }
 
-fn backtrace_printer<'a, K: DepKind>(
-    sess: &'a rustc_session::Session,
-    graph: &'a DepGraphData<K>,
-    node: SerializedDepNodeIndex,
-) -> OnDrop<impl Fn() + 'a> {
-    OnDrop(
-        #[inline(never)]
-        #[cold]
-        move || {
-            let node = graph.previous.index_to_node(node);
-            // Do not try to rely on DepNode's Debug implementation, since it may panic.
-            let diag = rustc_errors::Diagnostic::new(
-                rustc_errors::Level::FailureNote,
-                &format!(
-                    "encountered while trying to mark dependency green: {:?}({})",
-                    node.kind, node.hash
-                ),
-            );
-            sess.diagnostic().force_print_diagnostic(diag);
-        },
-    )
+#[inline(never)]
+#[cold]
+pub(crate) fn print_markframe_trace<K: DepKind>(
+    graph: &DepGraph<K>,
+    frame: Option<&MarkFrame<'_>>,
+) {
+    let data = graph.data.as_ref().unwrap();
+
+    eprintln!("there was a panic while trying to force a dep node");
+    eprintln!("try_mark_green dep node stack:");
+
+    let mut i = 0;
+    let mut current = frame;
+    while let Some(frame) = current {
+        let node = data.previous.index_to_node(frame.index);
+        eprintln!("#{i} {:?}", node);
+        current = frame.parent;
+        i += 1;
+    }
+
+    eprintln!("end of try_mark_green dep node stack");
 }
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index 6969f2dbe..40e713198 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -6,7 +6,8 @@ mod serialized;
 
 pub use dep_node::{DepKindStruct, DepNode, DepNodeParams, WorkProductId};
 pub use graph::{
-    hash_result, DepGraph, DepNodeColor, DepNodeIndex, TaskDeps, TaskDepsRef, WorkProduct,
+    hash_result, DepGraph, DepGraphData, DepNodeColor, DepNodeIndex, TaskDeps, TaskDepsRef,
+    WorkProduct,
 };
 pub use query::DepGraphQuery;
 pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex};
@@ -16,8 +17,10 @@ use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_serialize::{opaque::FileEncoder, Encodable};
 use rustc_session::Session;
 
-use std::fmt;
 use std::hash::Hash;
+use std::{fmt, panic};
+
+use self::graph::{print_markframe_trace, MarkFrame};
 
 pub trait DepContext: Copy {
     type DepKind: self::DepKind;
@@ -52,11 +55,23 @@ pub trait DepContext: Copy {
     }
 
     /// Try to force a dep node to execute and see if it's green.
-    #[instrument(skip(self), level = "debug")]
-    fn try_force_from_dep_node(self, dep_node: DepNode<Self::DepKind>) -> bool {
+    #[inline]
+    #[instrument(skip(self, frame), level = "debug")]
+    fn try_force_from_dep_node(
+        self,
+        dep_node: DepNode<Self::DepKind>,
+        frame: Option<&MarkFrame<'_>>,
+    ) -> bool {
         let cb = self.dep_kind_info(dep_node.kind);
         if let Some(f) = cb.force_from_dep_node {
-            f(self, dep_node);
+            if let Err(value) = panic::catch_unwind(panic::AssertUnwindSafe(|| {
+                f(self, dep_node);
+            })) {
+                if !value.is::<rustc_errors::FatalErrorMarker>() {
+                    print_markframe_trace(self.dep_graph(), frame);
+                }
+                panic::resume_unwind(value)
+            }
             true
         } else {
             false
@@ -88,6 +103,15 @@ impl<T: DepContext> HasDepContext for T {
     }
 }
 
+impl<T: HasDepContext, Q: Copy> HasDepContext for (T, Q) {
+    type DepKind = T::DepKind;
+    type DepContext = T::DepContext;
+
+    fn dep_context(&self) -> &Self::DepContext {
+        self.0.dep_context()
+    }
+}
+
 /// Describes the contents of the fingerprint generated by a given query.
 #[derive(Debug, PartialEq, Eq, Copy, Clone)]
 pub enum FingerprintStyle {
diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs
index 29513df46..3d19a8491 100644
--- a/compiler/rustc_query_system/src/dep_graph/serialized.rs
+++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs
@@ -80,11 +80,6 @@ impl<K: DepKind> SerializedDepGraph<K> {
     }
 
     #[inline]
-    pub fn fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
-        self.index.get(dep_node).map(|&node_index| self.fingerprints[node_index])
-    }
-
-    #[inline]
     pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint {
         self.fingerprints[dep_node_index]
     }
diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs
index 6cc4c9a7e..bb812b006 100644
--- a/compiler/rustc_query_system/src/lib.rs
+++ b/compiler/rustc_query_system/src/lib.rs
@@ -30,4 +30,4 @@ pub use error::LayoutOfDepth;
 pub use error::QueryOverflow;
 pub use values::Value;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index 4b3cd16c2..29f6a07e8 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -16,17 +16,11 @@ pub trait CacheSelector<'tcx, V> {
         V: Copy;
 }
 
-pub trait QueryStorage {
-    type Value: Copy;
-}
-
-pub trait QueryCache: QueryStorage + Sized {
+pub trait QueryCache: Sized {
     type Key: Hash + Eq + Copy + Debug;
+    type Value: Copy;
 
     /// Checks if the query is already computed and in the cache.
-    /// It returns the shard index and a lock guard to the shard,
-    /// which will be used if the query is not in the cache and we need
-    /// to compute it.
     fn lookup(&self, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>;
 
     fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex);
@@ -55,16 +49,13 @@ impl<K, V> Default for DefaultCache<K, V> {
     }
 }
 
-impl<K: Eq + Hash, V: Copy + Debug> QueryStorage for DefaultCache<K, V> {
-    type Value = V;
-}
-
 impl<K, V> QueryCache for DefaultCache<K, V>
 where
     K: Eq + Hash + Copy + Debug,
-    V: Copy + Debug,
+    V: Copy,
 {
     type Key = K;
+    type Value = V;
 
     #[inline(always)]
     fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
@@ -127,15 +118,12 @@ impl<V> Default for SingleCache<V> {
     }
 }
 
-impl<V: Copy + Debug> QueryStorage for SingleCache<V> {
-    type Value = V;
-}
-
 impl<V> QueryCache for SingleCache<V>
 where
-    V: Copy + Debug,
+    V: Copy,
 {
     type Key = ();
+    type Value = V;
 
     #[inline(always)]
     fn lookup(&self, _key: &()) -> Option<(V, DepNodeIndex)> {
@@ -148,7 +136,9 @@ where
     }
 
     fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
-        self.cache.lock().as_ref().map(|value| f(&(), &value.0, value.1));
+        if let Some(value) = self.cache.lock().as_ref() {
+            f(&(), &value.0, value.1)
+        }
     }
 }
 
@@ -173,16 +163,13 @@ impl<K: Idx, V> Default for VecCache<K, V> {
     }
 }
 
-impl<K: Eq + Idx, V: Copy + Debug> QueryStorage for VecCache<K, V> {
-    type Value = V;
-}
-
 impl<K, V> QueryCache for VecCache<K, V>
 where
     K: Eq + Idx + Copy + Debug,
-    V: Copy + Debug,
+    V: Copy,
 {
     type Key = K;
+    type Value = V;
 
     #[inline(always)]
     fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs
index d56373873..c8d779385 100644
--- a/compiler/rustc_query_system/src/query/config.rs
+++ b/compiler/rustc_query_system/src/query/config.rs
@@ -4,59 +4,67 @@ use crate::dep_graph::{DepNode, DepNodeParams, SerializedDepNodeIndex};
 use crate::error::HandleCycleError;
 use crate::ich::StableHashingContext;
 use crate::query::caches::QueryCache;
-use crate::query::{QueryContext, QueryState};
+use crate::query::{QueryContext, QueryInfo, QueryState};
 
 use rustc_data_structures::fingerprint::Fingerprint;
 use std::fmt::Debug;
 use std::hash::Hash;
 
-pub type HashResult<Qcx, Q> =
-    Option<fn(&mut StableHashingContext<'_>, &<Q as QueryConfig<Qcx>>::Value) -> Fingerprint>;
+pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
-pub type TryLoadFromDisk<Qcx, Q> =
-    Option<fn(Qcx, SerializedDepNodeIndex) -> Option<<Q as QueryConfig<Qcx>>::Value>>;
+pub type TryLoadFromDisk<Qcx, V> = Option<fn(Qcx, SerializedDepNodeIndex) -> Option<V>>;
 
-pub trait QueryConfig<Qcx: QueryContext> {
-    const NAME: &'static str;
+pub trait QueryConfig<Qcx: QueryContext>: Copy {
+    fn name(self) -> &'static str;
 
     // `Key` and `Value` are `Copy` instead of `Clone` to ensure copying them stays cheap,
     // but it isn't necessary.
     type Key: DepNodeParams<Qcx::DepContext> + Eq + Hash + Copy + Debug;
-    type Value: Debug + Copy;
+    type Value: Copy;
 
     type Cache: QueryCache<Key = Self::Key, Value = Self::Value>;
 
+    fn format_value(self) -> fn(&Self::Value) -> String;
+
     // Don't use this method to access query results, instead use the methods on TyCtxt
-    fn query_state<'a>(tcx: Qcx) -> &'a QueryState<Self::Key, Qcx::DepKind>
+    fn query_state<'a>(self, tcx: Qcx) -> &'a QueryState<Self::Key, Qcx::DepKind>
     where
         Qcx: 'a;
 
     // Don't use this method to access query results, instead use the methods on TyCtxt
-    fn query_cache<'a>(tcx: Qcx) -> &'a Self::Cache
+    fn query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache
     where
         Qcx: 'a;
 
-    fn cache_on_disk(tcx: Qcx::DepContext, key: &Self::Key) -> bool;
+    fn cache_on_disk(self, tcx: Qcx::DepContext, key: &Self::Key) -> bool;
 
     // Don't use this method to compute query results, instead use the methods on TyCtxt
-    fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Value;
+    fn execute_query(self, tcx: Qcx::DepContext, k: Self::Key) -> Self::Value;
+
+    fn compute(self, tcx: Qcx, key: Self::Key) -> Self::Value;
 
-    fn compute(tcx: Qcx, key: Self::Key) -> Self::Value;
+    fn try_load_from_disk(self, qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk<Qcx, Self::Value>;
 
-    fn try_load_from_disk(qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk<Qcx, Self>;
+    fn loadable_from_disk(self, qcx: Qcx, key: &Self::Key, idx: SerializedDepNodeIndex) -> bool;
 
-    const ANON: bool;
-    const EVAL_ALWAYS: bool;
-    const DEPTH_LIMIT: bool;
-    const FEEDABLE: bool;
+    /// Synthesize an error value to let compilation continue after a cycle.
+    fn value_from_cycle_error(
+        self,
+        tcx: Qcx::DepContext,
+        cycle: &[QueryInfo<Qcx::DepKind>],
+    ) -> Self::Value;
 
-    const DEP_KIND: Qcx::DepKind;
-    const HANDLE_CYCLE_ERROR: HandleCycleError;
+    fn anon(self) -> bool;
+    fn eval_always(self) -> bool;
+    fn depth_limit(self) -> bool;
+    fn feedable(self) -> bool;
 
-    const HASH_RESULT: HashResult<Qcx, Self>;
+    fn dep_kind(self) -> Qcx::DepKind;
+    fn handle_cycle_error(self) -> HandleCycleError;
+    fn hash_result(self) -> HashResult<Self::Value>;
 
     // Just here for convernience and checking that the key matches the kind, don't override this.
-    fn construct_dep_node(tcx: Qcx::DepContext, key: &Self::Key) -> DepNode<Qcx::DepKind> {
-        DepNode::construct(tcx, Self::DEP_KIND, key)
+    fn construct_dep_node(self, tcx: Qcx::DepContext, key: &Self::Key) -> DepNode<Qcx::DepKind> {
+        DepNode::construct(tcx, self.dep_kind(), key)
     }
 }
diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index a5a2f0093..a534b5407 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -18,11 +18,11 @@ use std::num::NonZeroU64;
 #[cfg(parallel_compiler)]
 use {
     parking_lot::{Condvar, Mutex},
+    rayon_core,
     rustc_data_structures::fx::FxHashSet,
     rustc_data_structures::sync::Lock,
     rustc_data_structures::sync::Lrc,
     rustc_data_structures::{jobserver, OnDrop},
-    rustc_rayon_core as rayon_core,
     rustc_span::DUMMY_SP,
     std::iter,
     std::process,
@@ -124,8 +124,6 @@ impl<D: DepKind> QueryJob<D> {
 }
 
 impl QueryJobId {
-    #[cold]
-    #[inline(never)]
     #[cfg(not(parallel_compiler))]
     pub(super) fn find_cycle_in_stack<D: DepKind>(
         &self,
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 383c63cd2..312b0e168 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -8,8 +8,7 @@ pub use self::job::{print_query_stack, QueryInfo, QueryJob, QueryJobId, QueryJob
 
 mod caches;
 pub use self::caches::{
-    CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage, SingleCacheSelector,
-    VecCacheSelector,
+    CacheSelector, DefaultCacheSelector, QueryCache, SingleCacheSelector, VecCacheSelector,
 };
 
 mod config;
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index 5f003fa70..20310483d 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -3,22 +3,22 @@
 //! manage the caches, and so forth.
 
 use crate::dep_graph::{DepContext, DepKind, DepNode, DepNodeIndex, DepNodeParams};
+use crate::dep_graph::{DepGraphData, HasDepContext};
 use crate::ich::StableHashingContext;
 use crate::query::caches::QueryCache;
+#[cfg(parallel_compiler)]
+use crate::query::job::QueryLatch;
 use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
+use crate::query::SerializedDepNodeIndex;
 use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
-use crate::values::Value;
 use crate::HandleCycleError;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxHashMap;
-#[cfg(parallel_compiler)]
-use rustc_data_structures::profiling::TimingGuard;
-#[cfg(parallel_compiler)]
-use rustc_data_structures::sharded::Sharded;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::sync::Lock;
+#[cfg(parallel_compiler)]
+use rustc_data_structures::{cold_path, sharded::Sharded};
 use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError};
-use rustc_session::Session;
 use rustc_span::{Span, DUMMY_SP};
 use std::cell::Cell;
 use std::collections::hash_map::Entry;
@@ -115,48 +115,49 @@ where
 {
     state: &'tcx QueryState<K, D>,
     key: K,
-    id: QueryJobId,
 }
 
 #[cold]
 #[inline(never)]
-fn mk_cycle<Qcx, R, D: DepKind>(
+fn mk_cycle<Q, Qcx>(
+    query: Q,
     qcx: Qcx,
-    cycle_error: CycleError<D>,
+    cycle_error: CycleError<Qcx::DepKind>,
     handler: HandleCycleError,
-) -> R
+) -> Q::Value
 where
-    Qcx: QueryContext + crate::query::HasDepContext<DepKind = D>,
-    R: std::fmt::Debug + Value<Qcx::DepContext, Qcx::DepKind>,
+    Q: QueryConfig<Qcx>,
+    Qcx: QueryContext,
 {
     let error = report_cycle(qcx.dep_context().sess(), &cycle_error);
-    handle_cycle_error(*qcx.dep_context(), &cycle_error, error, handler)
+    handle_cycle_error(query, qcx, &cycle_error, error, handler)
 }
 
-fn handle_cycle_error<Tcx, V>(
-    tcx: Tcx,
-    cycle_error: &CycleError<Tcx::DepKind>,
+fn handle_cycle_error<Q, Qcx>(
+    query: Q,
+    qcx: Qcx,
+    cycle_error: &CycleError<Qcx::DepKind>,
     mut error: DiagnosticBuilder<'_, ErrorGuaranteed>,
     handler: HandleCycleError,
-) -> V
+) -> Q::Value
 where
-    Tcx: DepContext,
-    V: Value<Tcx, Tcx::DepKind>,
+    Q: QueryConfig<Qcx>,
+    Qcx: QueryContext,
 {
     use HandleCycleError::*;
     match handler {
         Error => {
             error.emit();
-            Value::from_cycle_error(tcx, &cycle_error.cycle)
+            query.value_from_cycle_error(*qcx.dep_context(), &cycle_error.cycle)
         }
         Fatal => {
             error.emit();
-            tcx.sess().abort_if_errors();
+            qcx.dep_context().sess().abort_if_errors();
             unreachable!()
         }
         DelayBug => {
             error.delay_as_bug();
-            Value::from_cycle_error(tcx, &cycle_error.cycle)
+            query.value_from_cycle_error(*qcx.dep_context(), &cycle_error.cycle)
         }
     }
 }
@@ -165,84 +166,6 @@ impl<'tcx, K, D: DepKind> JobOwner<'tcx, K, D>
 where
     K: Eq + Hash + Copy,
 {
-    /// Either gets a `JobOwner` corresponding the query, allowing us to
-    /// start executing the query, or returns with the result of the query.
-    /// This function assumes that `try_get_cached` is already called and returned `lookup`.
-    /// If the query is executing elsewhere, this will wait for it and return the result.
-    /// If the query panicked, this will silently panic.
-    ///
-    /// This function is inlined because that results in a noticeable speed-up
-    /// for some compile-time benchmarks.
-    #[inline(always)]
-    fn try_start<'b, Qcx>(
-        qcx: &'b Qcx,
-        state: &'b QueryState<K, Qcx::DepKind>,
-        span: Span,
-        key: K,
-    ) -> TryGetJob<'b, K, D>
-    where
-        Qcx: QueryContext + crate::query::HasDepContext<DepKind = D>,
-    {
-        #[cfg(parallel_compiler)]
-        let mut state_lock = state.active.get_shard_by_value(&key).lock();
-        #[cfg(not(parallel_compiler))]
-        let mut state_lock = state.active.lock();
-        let lock = &mut *state_lock;
-        let current_job_id = qcx.current_query_job();
-
-        match lock.entry(key) {
-            Entry::Vacant(entry) => {
-                let id = qcx.next_job_id();
-                let job = QueryJob::new(id, span, current_job_id);
-
-                let key = *entry.key();
-                entry.insert(QueryResult::Started(job));
-
-                let owner = JobOwner { state, id, key };
-                return TryGetJob::NotYetStarted(owner);
-            }
-            Entry::Occupied(mut entry) => {
-                match entry.get_mut() {
-                    #[cfg(not(parallel_compiler))]
-                    QueryResult::Started(job) => {
-                        let id = job.id;
-                        drop(state_lock);
-
-                        // If we are single-threaded we know that we have cycle error,
-                        // so we just return the error.
-                        return TryGetJob::Cycle(id.find_cycle_in_stack(
-                            qcx.try_collect_active_jobs().unwrap(),
-                            &current_job_id,
-                            span,
-                        ));
-                    }
-                    #[cfg(parallel_compiler)]
-                    QueryResult::Started(job) => {
-                        // For parallel queries, we'll block and wait until the query running
-                        // in another thread has completed. Record how long we wait in the
-                        // self-profiler.
-                        let query_blocked_prof_timer = qcx.dep_context().profiler().query_blocked();
-
-                        // Get the latch out
-                        let latch = job.latch();
-
-                        drop(state_lock);
-
-                        // With parallel queries we might just have to wait on some other
-                        // thread.
-                        let result = latch.wait_on(current_job_id, span);
-
-                        match result {
-                            Ok(()) => TryGetJob::JobCompleted(query_blocked_prof_timer),
-                            Err(cycle) => TryGetJob::Cycle(cycle),
-                        }
-                    }
-                    QueryResult::Poisoned => FatalError.raise(),
-                }
-            }
-        }
-    }
-
     /// Completes the query by updating the query cache with the `result`,
     /// signals the waiter and forgets the JobOwner, so it won't poison the query
     fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex)
@@ -309,25 +232,6 @@ pub(crate) struct CycleError<D: DepKind> {
     pub cycle: Vec<QueryInfo<D>>,
 }
 
-/// The result of `try_start`.
-enum TryGetJob<'tcx, K, D>
-where
-    K: Eq + Hash + Copy,
-    D: DepKind,
-{
-    /// The query is not yet started. Contains a guard to the cache eventually used to start it.
-    NotYetStarted(JobOwner<'tcx, K, D>),
-
-    /// The query was already completed.
-    /// Returns the result of the query and its dep-node index
-    /// if it succeeded or a cycle error if it failed.
-    #[cfg(parallel_compiler)]
-    JobCompleted(TimingGuard<'tcx>),
-
-    /// Trying to execute the query resulted in a cycle.
-    Cycle(CycleError<D>),
-}
-
 /// Checks if the query is already computed and in the cache.
 /// It returns the shard index and a lock guard to the shard,
 /// which will be used if the query is not in the cache and we need
@@ -348,44 +252,54 @@ where
     }
 }
 
+#[cold]
 #[inline(never)]
-fn try_execute_query<Q, Qcx>(
+#[cfg(not(parallel_compiler))]
+fn cycle_error<Q, Qcx>(
+    query: Q,
+    qcx: Qcx,
+    try_execute: QueryJobId,
+    span: Span,
+) -> (Q::Value, Option<DepNodeIndex>)
+where
+    Q: QueryConfig<Qcx>,
+    Qcx: QueryContext,
+{
+    let error = try_execute.find_cycle_in_stack(
+        qcx.try_collect_active_jobs().unwrap(),
+        &qcx.current_query_job(),
+        span,
+    );
+    (mk_cycle(query, qcx, error, query.handle_cycle_error()), None)
+}
+
+#[inline(always)]
+#[cfg(parallel_compiler)]
+fn wait_for_query<Q, Qcx>(
+    query: Q,
     qcx: Qcx,
     span: Span,
     key: Q::Key,
-    dep_node: Option<DepNode<Qcx::DepKind>>,
+    latch: QueryLatch<Qcx::DepKind>,
+    current: Option<QueryJobId>,
 ) -> (Q::Value, Option<DepNodeIndex>)
 where
     Q: QueryConfig<Qcx>,
     Qcx: QueryContext,
 {
-    let state = Q::query_state(qcx);
-    match JobOwner::<'_, Q::Key, Qcx::DepKind>::try_start(&qcx, state, span, key) {
-        TryGetJob::NotYetStarted(job) => {
-            let (result, dep_node_index) = execute_job::<Q, Qcx>(qcx, key, dep_node, job.id);
-            let cache = Q::query_cache(qcx);
-            if Q::FEEDABLE {
-                // We should not compute queries that also got a value via feeding.
-                // This can't happen, as query feeding adds the very dependencies to the fed query
-                // as its feeding query had. So if the fed query is red, so is its feeder, which will
-                // get evaluated first, and re-feed the query.
-                if let Some((cached_result, _)) = cache.lookup(&key) {
-                    panic!(
-                        "fed query later has its value computed. The already cached value: {cached_result:?}"
-                    );
-                }
-            }
-            job.complete(cache, result, dep_node_index);
-            (result, Some(dep_node_index))
-        }
-        TryGetJob::Cycle(error) => {
-            let result = mk_cycle(qcx, error, Q::HANDLE_CYCLE_ERROR);
-            (result, None)
-        }
-        #[cfg(parallel_compiler)]
-        TryGetJob::JobCompleted(query_blocked_prof_timer) => {
-            let Some((v, index)) = Q::query_cache(qcx).lookup(&key) else {
-                panic!("value must be in cache after waiting")
+    // For parallel queries, we'll block and wait until the query running
+    // in another thread has completed. Record how long we wait in the
+    // self-profiler.
+    let query_blocked_prof_timer = qcx.dep_context().profiler().query_blocked();
+
+    // With parallel queries we might just have to wait on some other
+    // thread.
+    let result = latch.wait_on(current, span);
+
+    match result {
+        Ok(()) => {
+            let Some((v, index)) = query.query_cache(qcx).lookup(&key) else {
+                cold_path(|| panic!("value must be in cache after waiting"))
             };
 
             qcx.dep_context().profiler().query_cache_hit(index.into());
@@ -393,57 +307,178 @@ where
 
             (v, Some(index))
         }
+        Err(cycle) => (mk_cycle(query, qcx, cycle, query.handle_cycle_error()), None),
+    }
+}
+
+#[inline(never)]
+fn try_execute_query<Q, Qcx>(
+    query: Q,
+    qcx: Qcx,
+    span: Span,
+    key: Q::Key,
+    dep_node: Option<DepNode<Qcx::DepKind>>,
+) -> (Q::Value, Option<DepNodeIndex>)
+where
+    Q: QueryConfig<Qcx>,
+    Qcx: QueryContext,
+{
+    let state = query.query_state(qcx);
+    #[cfg(parallel_compiler)]
+    let mut state_lock = state.active.get_shard_by_value(&key).lock();
+    #[cfg(not(parallel_compiler))]
+    let mut state_lock = state.active.lock();
+
+    // For the parallel compiler we need to check both the query cache and query state structures
+    // while holding the state lock to ensure that 1) the query has not yet completed and 2) the
+    // query is not still executing. Without checking the query cache here, we can end up
+    // re-executing the query since `try_start` only checks that the query is not currently
+    // executing, but another thread may have already completed the query and stores it result
+    // in the query cache.
+    if cfg!(parallel_compiler) && qcx.dep_context().sess().threads() > 1 {
+        if let Some((value, index)) = query.query_cache(qcx).lookup(&key) {
+            qcx.dep_context().profiler().query_cache_hit(index.into());
+            return (value, Some(index));
+        }
+    }
+
+    let current_job_id = qcx.current_query_job();
+
+    match state_lock.entry(key) {
+        Entry::Vacant(entry) => {
+            // Nothing has computed or is computing the query, so we start a new job and insert it in the
+            // state map.
+            let id = qcx.next_job_id();
+            let job = QueryJob::new(id, span, current_job_id);
+            entry.insert(QueryResult::Started(job));
+
+            // Drop the lock before we start executing the query
+            drop(state_lock);
+
+            execute_job(query, qcx, state, key, id, dep_node)
+        }
+        Entry::Occupied(mut entry) => {
+            match entry.get_mut() {
+                #[cfg(not(parallel_compiler))]
+                QueryResult::Started(job) => {
+                    let id = job.id;
+                    drop(state_lock);
+
+                    // If we are single-threaded we know that we have cycle error,
+                    // so we just return the error.
+                    cycle_error(query, qcx, id, span)
+                }
+                #[cfg(parallel_compiler)]
+                QueryResult::Started(job) => {
+                    // Get the latch out
+                    let latch = job.latch();
+                    drop(state_lock);
+
+                    wait_for_query(query, qcx, span, key, latch, current_job_id)
+                }
+                QueryResult::Poisoned => FatalError.raise(),
+            }
+        }
     }
 }
 
 #[inline(always)]
 fn execute_job<Q, Qcx>(
+    query: Q,
     qcx: Qcx,
+    state: &QueryState<Q::Key, Qcx::DepKind>,
     key: Q::Key,
-    mut dep_node_opt: Option<DepNode<Qcx::DepKind>>,
-    job_id: QueryJobId,
-) -> (Q::Value, DepNodeIndex)
+    id: QueryJobId,
+    dep_node: Option<DepNode<Qcx::DepKind>>,
+) -> (Q::Value, Option<DepNodeIndex>)
 where
     Q: QueryConfig<Qcx>,
     Qcx: QueryContext,
 {
-    let dep_graph = qcx.dep_context().dep_graph();
+    // Use `JobOwner` so the query will be poisoned if executing it panics.
+    let job_owner = JobOwner { state, key };
+
+    let (result, dep_node_index) = match qcx.dep_context().dep_graph().data() {
+        None => execute_job_non_incr(query, qcx, key, id),
+        Some(data) => execute_job_incr(query, qcx, data, key, dep_node, id),
+    };
 
-    // Fast path for when incr. comp. is off.
-    if !dep_graph.is_fully_enabled() {
-        // Fingerprint the key, just to assert that it doesn't
-        // have anything we don't consider hashable
-        if cfg!(debug_assertions) {
-            let _ = key.to_fingerprint(*qcx.dep_context());
+    let cache = query.query_cache(qcx);
+    if query.feedable() {
+        // We should not compute queries that also got a value via feeding.
+        // This can't happen, as query feeding adds the very dependencies to the fed query
+        // as its feeding query had. So if the fed query is red, so is its feeder, which will
+        // get evaluated first, and re-feed the query.
+        if let Some((cached_result, _)) = cache.lookup(&key) {
+            panic!(
+                "fed query later has its value computed. The already cached value: {}",
+                (query.format_value())(&cached_result)
+            );
         }
+    }
+    job_owner.complete(cache, result, dep_node_index);
 
-        let prof_timer = qcx.dep_context().profiler().query_provider();
-        let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || Q::compute(qcx, key));
-        let dep_node_index = dep_graph.next_virtual_depnode_index();
-        prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+    (result, Some(dep_node_index))
+}
 
-        // Similarly, fingerprint the result to assert that
-        // it doesn't have anything not considered hashable.
-        if cfg!(debug_assertions)
-            && let Some(hash_result) = Q::HASH_RESULT
-        {
-            qcx.dep_context().with_stable_hashing_context(|mut hcx| {
-                hash_result(&mut hcx, &result);
-            });
-        }
+// Fast path for when incr. comp. is off.
+#[inline(always)]
+fn execute_job_non_incr<Q, Qcx>(
+    query: Q,
+    qcx: Qcx,
+    key: Q::Key,
+    job_id: QueryJobId,
+) -> (Q::Value, DepNodeIndex)
+where
+    Q: QueryConfig<Qcx>,
+    Qcx: QueryContext,
+{
+    debug_assert!(!qcx.dep_context().dep_graph().is_fully_enabled());
 
-        return (result, dep_node_index);
+    // Fingerprint the key, just to assert that it doesn't
+    // have anything we don't consider hashable
+    if cfg!(debug_assertions) {
+        let _ = key.to_fingerprint(*qcx.dep_context());
     }
 
-    if !Q::ANON && !Q::EVAL_ALWAYS {
+    let prof_timer = qcx.dep_context().profiler().query_provider();
+    let result = qcx.start_query(job_id, query.depth_limit(), None, || query.compute(qcx, key));
+    let dep_node_index = qcx.dep_context().dep_graph().next_virtual_depnode_index();
+    prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+    // Similarly, fingerprint the result to assert that
+    // it doesn't have anything not considered hashable.
+    if cfg!(debug_assertions) && let Some(hash_result) = query.hash_result() {
+        qcx.dep_context().with_stable_hashing_context(|mut hcx| {
+            hash_result(&mut hcx, &result);
+        });
+    }
+
+    (result, dep_node_index)
+}
+
+#[inline(always)]
+fn execute_job_incr<Q, Qcx>(
+    query: Q,
+    qcx: Qcx,
+    dep_graph_data: &DepGraphData<Qcx::DepKind>,
+    key: Q::Key,
+    mut dep_node_opt: Option<DepNode<Qcx::DepKind>>,
+    job_id: QueryJobId,
+) -> (Q::Value, DepNodeIndex)
+where
+    Q: QueryConfig<Qcx>,
+    Qcx: QueryContext,
+{
+    if !query.anon() && !query.eval_always() {
         // `to_dep_node` is expensive for some `DepKind`s.
         let dep_node =
-            dep_node_opt.get_or_insert_with(|| Q::construct_dep_node(*qcx.dep_context(), &key));
+            dep_node_opt.get_or_insert_with(|| query.construct_dep_node(*qcx.dep_context(), &key));
 
         // The diagnostics for this query will be promoted to the current session during
         // `try_mark_green()`, so we can ignore them here.
         if let Some(ret) = qcx.start_query(job_id, false, None, || {
-            try_load_from_disk_and_cache_in_memory::<Q, Qcx>(qcx, &key, &dep_node)
+            try_load_from_disk_and_cache_in_memory(query, dep_graph_data, qcx, &key, &dep_node)
         }) {
             return ret;
         }
@@ -453,17 +488,24 @@ where
     let diagnostics = Lock::new(ThinVec::new());
 
     let (result, dep_node_index) =
-        qcx.start_query(job_id, Q::DEPTH_LIMIT, Some(&diagnostics), || {
-            if Q::ANON {
-                return dep_graph
-                    .with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || Q::compute(qcx, key));
+        qcx.start_query(job_id, query.depth_limit(), Some(&diagnostics), || {
+            if query.anon() {
+                return dep_graph_data.with_anon_task(*qcx.dep_context(), query.dep_kind(), || {
+                    query.compute(qcx, key)
+                });
             }
 
             // `to_dep_node` is expensive for some `DepKind`s.
             let dep_node =
-                dep_node_opt.unwrap_or_else(|| Q::construct_dep_node(*qcx.dep_context(), &key));
-
-            dep_graph.with_task(dep_node, qcx, key, Q::compute, Q::HASH_RESULT)
+                dep_node_opt.unwrap_or_else(|| query.construct_dep_node(*qcx.dep_context(), &key));
+
+            dep_graph_data.with_task(
+                dep_node,
+                (qcx, query),
+                key,
+                |(qcx, query), key| query.compute(qcx, key),
+                query.hash_result(),
+            )
         });
 
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@@ -472,7 +514,7 @@ where
     let side_effects = QuerySideEffects { diagnostics };
 
     if std::intrinsics::unlikely(!side_effects.is_empty()) {
-        if Q::ANON {
+        if query.anon() {
             qcx.store_side_effects_for_anon_node(dep_node_index, side_effects);
         } else {
             qcx.store_side_effects(dep_node_index, side_effects);
@@ -484,6 +526,8 @@ where
 
 #[inline(always)]
 fn try_load_from_disk_and_cache_in_memory<Q, Qcx>(
+    query: Q,
+    dep_graph_data: &DepGraphData<Qcx::DepKind>,
     qcx: Qcx,
     key: &Q::Key,
     dep_node: &DepNode<Qcx::DepKind>,
@@ -495,21 +539,22 @@ where
     // Note this function can be called concurrently from the same query
     // We must ensure that this is handled correctly.
 
-    let dep_graph = qcx.dep_context().dep_graph();
-    let (prev_dep_node_index, dep_node_index) = dep_graph.try_mark_green(qcx, &dep_node)?;
+    let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?;
 
-    debug_assert!(dep_graph.is_green(dep_node));
+    debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index));
 
     // First we try to load the result from the on-disk cache.
     // Some things are never cached on disk.
-    if let Some(try_load_from_disk) = Q::try_load_from_disk(qcx, &key) {
+    if let Some(try_load_from_disk) = query.try_load_from_disk(qcx, &key) {
         let prof_timer = qcx.dep_context().profiler().incr_cache_loading();
 
         // The call to `with_query_deserialization` enforces that no new `DepNodes`
         // are created during deserialization. See the docs of that method for more
         // details.
-        let result =
-            dep_graph.with_query_deserialization(|| try_load_from_disk(qcx, prev_dep_node_index));
+        let result = qcx
+            .dep_context()
+            .dep_graph()
+            .with_query_deserialization(|| try_load_from_disk(qcx, prev_dep_node_index));
 
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -517,14 +562,10 @@ where
             if std::intrinsics::unlikely(
                 qcx.dep_context().sess().opts.unstable_opts.query_dep_graph,
             ) {
-                dep_graph.mark_debug_loaded_from_disk(*dep_node)
+                dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
             }
 
-            let prev_fingerprint = qcx
-                .dep_context()
-                .dep_graph()
-                .prev_fingerprint_of(dep_node)
-                .unwrap_or(Fingerprint::ZERO);
+            let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index);
             // If `-Zincremental-verify-ich` is specified, re-hash results from
             // the cache and make sure that they have the expected fingerprint.
             //
@@ -536,7 +577,14 @@ where
             if std::intrinsics::unlikely(
                 try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich,
             ) {
-                incremental_verify_ich(*qcx.dep_context(), &result, dep_node, Q::HASH_RESULT);
+                incremental_verify_ich(
+                    *qcx.dep_context(),
+                    dep_graph_data,
+                    &result,
+                    prev_dep_node_index,
+                    query.hash_result(),
+                    query.format_value(),
+                );
             }
 
             return Some((result, dep_node_index));
@@ -546,16 +594,23 @@ where
         // can be forced from `DepNode`.
         debug_assert!(
             !qcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(),
-            "missing on-disk cache entry for {dep_node:?}"
+            "missing on-disk cache entry for reconstructible {dep_node:?}"
         );
     }
 
+    // Sanity check for the logic in `ensure`: if the node is green and the result loadable,
+    // we should actually be able to load it.
+    debug_assert!(
+        !query.loadable_from_disk(qcx, &key, prev_dep_node_index),
+        "missing on-disk cache entry for loadable {dep_node:?}"
+    );
+
     // We could not load a result from the on-disk cache, so
     // recompute.
     let prof_timer = qcx.dep_context().profiler().query_provider();
 
     // The dep-graph for this computation is already in-place.
-    let result = dep_graph.with_ignore(|| Q::compute(qcx, *key));
+    let result = qcx.dep_context().dep_graph().with_ignore(|| query.compute(qcx, *key));
 
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -568,87 +623,69 @@ where
     //
     // See issue #82920 for an example of a miscompilation that would get turned into
     // an ICE by this check
-    incremental_verify_ich(*qcx.dep_context(), &result, dep_node, Q::HASH_RESULT);
+    incremental_verify_ich(
+        *qcx.dep_context(),
+        dep_graph_data,
+        &result,
+        prev_dep_node_index,
+        query.hash_result(),
+        query.format_value(),
+    );
 
     Some((result, dep_node_index))
 }
 
 #[inline]
-#[instrument(skip(tcx, result, hash_result), level = "debug")]
-pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
+#[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")]
+pub(crate) fn incremental_verify_ich<Tcx, V>(
     tcx: Tcx,
+    dep_graph_data: &DepGraphData<Tcx::DepKind>,
     result: &V,
-    dep_node: &DepNode<Tcx::DepKind>,
+    prev_index: SerializedDepNodeIndex,
     hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
-) -> Fingerprint
-where
+    format_value: fn(&V) -> String,
+) where
     Tcx: DepContext,
 {
-    assert!(
-        tcx.dep_graph().is_green(dep_node),
-        "fingerprint for green query instance not loaded from cache: {dep_node:?}",
-    );
+    if !dep_graph_data.is_index_green(prev_index) {
+        incremental_verify_ich_not_green(tcx, prev_index)
+    }
 
     let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| {
         tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result))
     });
 
-    let old_hash = tcx.dep_graph().prev_fingerprint_of(dep_node);
+    let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
 
-    if Some(new_hash) != old_hash {
-        incremental_verify_ich_failed(
-            tcx.sess(),
-            DebugArg::from(&dep_node),
-            DebugArg::from(&result),
-        );
+    if new_hash != old_hash {
+        incremental_verify_ich_failed(tcx, prev_index, &|| format_value(&result));
     }
-
-    new_hash
-}
-
-// This DebugArg business is largely a mirror of std::fmt::ArgumentV1, which is
-// currently not exposed publicly.
-//
-// The PR which added this attempted to use `&dyn Debug` instead, but that
-// showed statistically significant worse compiler performance. It's not
-// actually clear what the cause there was -- the code should be cold. If this
-// can be replaced with `&dyn Debug` with on perf impact, then it probably
-// should be.
-extern "C" {
-    type Opaque;
-}
-
-struct DebugArg<'a> {
-    value: &'a Opaque,
-    fmt: fn(&Opaque, &mut std::fmt::Formatter<'_>) -> std::fmt::Result,
 }
 
-impl<'a, T> From<&'a T> for DebugArg<'a>
+#[cold]
+#[inline(never)]
+fn incremental_verify_ich_not_green<Tcx>(tcx: Tcx, prev_index: SerializedDepNodeIndex)
 where
-    T: std::fmt::Debug,
+    Tcx: DepContext,
 {
-    fn from(value: &'a T) -> DebugArg<'a> {
-        DebugArg {
-            value: unsafe { std::mem::transmute(value) },
-            fmt: unsafe {
-                std::mem::transmute(<T as std::fmt::Debug>::fmt as fn(_, _) -> std::fmt::Result)
-            },
-        }
-    }
+    panic!(
+        "fingerprint for green query instance not loaded from cache: {:?}",
+        tcx.dep_graph().data().unwrap().prev_node_of(prev_index)
+    )
 }
 
-impl std::fmt::Debug for DebugArg<'_> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        (self.fmt)(self.value, f)
-    }
-}
-
-// Note that this is marked #[cold] and intentionally takes the equivalent of
-// `dyn Debug` for its arguments, as we want to avoid generating a bunch of
-// different implementations for LLVM to chew on (and filling up the final
-// binary, too).
+// Note that this is marked #[cold] and intentionally takes `dyn Debug` for `result`,
+// as we want to avoid generating a bunch of different implementations for LLVM to
+// chew on (and filling up the final binary, too).
 #[cold]
-fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: DebugArg<'_>) {
+#[inline(never)]
+fn incremental_verify_ich_failed<Tcx>(
+    tcx: Tcx,
+    prev_index: SerializedDepNodeIndex,
+    result: &dyn Fn() -> String,
+) where
+    Tcx: DepContext,
+{
     // When we emit an error message and panic, we try to debug-print the `DepNode`
     // and query result. Unfortunately, this can cause us to run additional queries,
     // which may result in another fingerprint mismatch while we're in the middle
@@ -662,19 +699,20 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result:
     let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true));
 
     if old_in_panic {
-        sess.emit_err(crate::error::Reentrant);
+        tcx.sess().emit_err(crate::error::Reentrant);
     } else {
-        let run_cmd = if let Some(crate_name) = &sess.opts.crate_name {
+        let run_cmd = if let Some(crate_name) = &tcx.sess().opts.crate_name {
             format!("`cargo clean -p {crate_name}` or `cargo clean`")
         } else {
             "`cargo clean`".to_string()
         };
 
-        sess.emit_err(crate::error::IncrementCompilation {
+        let dep_node = tcx.dep_graph().data().unwrap().prev_node_of(prev_index);
+        tcx.sess().emit_err(crate::error::IncrementCompilation {
             run_cmd,
             dep_node: format!("{dep_node:?}"),
         });
-        panic!("Found unstable fingerprints for {dep_node:?}: {result:?}");
+        panic!("Found unstable fingerprints for {dep_node:?}: {}", result());
     }
 
     INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.set(old_in_panic));
@@ -689,22 +727,27 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result:
 ///
 /// Note: The optimization is only available during incr. comp.
 #[inline(never)]
-fn ensure_must_run<Q, Qcx>(qcx: Qcx, key: &Q::Key) -> (bool, Option<DepNode<Qcx::DepKind>>)
+fn ensure_must_run<Q, Qcx>(
+    query: Q,
+    qcx: Qcx,
+    key: &Q::Key,
+    check_cache: bool,
+) -> (bool, Option<DepNode<Qcx::DepKind>>)
 where
     Q: QueryConfig<Qcx>,
     Qcx: QueryContext,
 {
-    if Q::EVAL_ALWAYS {
+    if query.eval_always() {
         return (true, None);
     }
 
     // Ensuring an anonymous query makes no sense
-    assert!(!Q::ANON);
+    assert!(!query.anon());
 
-    let dep_node = Q::construct_dep_node(*qcx.dep_context(), key);
+    let dep_node = query.construct_dep_node(*qcx.dep_context(), key);
 
     let dep_graph = qcx.dep_context().dep_graph();
-    match dep_graph.try_mark_green(qcx, &dep_node) {
+    let serialized_dep_node_index = match dep_graph.try_mark_green(qcx, &dep_node) {
         None => {
             // A None return from `try_mark_green` means that this is either
             // a new dep node or that the dep node has already been marked red.
@@ -712,32 +755,44 @@ where
             // DepNodeIndex. We must invoke the query itself. The performance cost
             // this introduces should be negligible as we'll immediately hit the
             // in-memory cache, or another query down the line will.
-            (true, Some(dep_node))
+            return (true, Some(dep_node));
         }
-        Some((_, dep_node_index)) => {
+        Some((serialized_dep_node_index, dep_node_index)) => {
             dep_graph.read_index(dep_node_index);
             qcx.dep_context().profiler().query_cache_hit(dep_node_index.into());
-            (false, None)
+            serialized_dep_node_index
         }
+    };
+
+    // We do not need the value at all, so do not check the cache.
+    if !check_cache {
+        return (false, None);
     }
+
+    let loadable = query.loadable_from_disk(qcx, key, serialized_dep_node_index);
+    (!loadable, Some(dep_node))
 }
 
 #[derive(Debug)]
 pub enum QueryMode {
     Get,
-    Ensure,
+    Ensure { check_cache: bool },
 }
 
 #[inline(always)]
-pub fn get_query<Q, Qcx, D>(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Value>
+pub fn get_query<Q, Qcx>(
+    query: Q,
+    qcx: Qcx,
+    span: Span,
+    key: Q::Key,
+    mode: QueryMode,
+) -> Option<Q::Value>
 where
-    D: DepKind,
     Q: QueryConfig<Qcx>,
-    Q::Value: Value<Qcx::DepContext, D>,
     Qcx: QueryContext,
 {
-    let dep_node = if let QueryMode::Ensure = mode {
-        let (must_run, dep_node) = ensure_must_run::<Q, _>(qcx, &key);
+    let dep_node = if let QueryMode::Ensure { check_cache } = mode {
+        let (must_run, dep_node) = ensure_must_run(query, qcx, &key, check_cache);
         if !must_run {
             return None;
         }
@@ -747,28 +802,30 @@ where
     };
 
     let (result, dep_node_index) =
-        ensure_sufficient_stack(|| try_execute_query::<Q, Qcx>(qcx, span, key, dep_node));
+        ensure_sufficient_stack(|| try_execute_query(query, qcx, span, key, dep_node));
     if let Some(dep_node_index) = dep_node_index {
         qcx.dep_context().dep_graph().read_index(dep_node_index)
     }
     Some(result)
 }
 
-pub fn force_query<Q, Qcx, D>(qcx: Qcx, key: Q::Key, dep_node: DepNode<Qcx::DepKind>)
-where
-    D: DepKind,
+pub fn force_query<Q, Qcx>(
+    query: Q,
+    qcx: Qcx,
+    key: Q::Key,
+    dep_node: DepNode<<Qcx as HasDepContext>::DepKind>,
+) where
     Q: QueryConfig<Qcx>,
-    Q::Value: Value<Qcx::DepContext, D>,
     Qcx: QueryContext,
 {
     // We may be concurrently trying both execute and force a query.
     // Ensure that only one of them runs the query.
-    if let Some((_, index)) = Q::query_cache(qcx).lookup(&key) {
+    if let Some((_, index)) = query.query_cache(qcx).lookup(&key) {
         qcx.dep_context().profiler().query_cache_hit(index.into());
         return;
     }
 
-    debug_assert!(!Q::ANON);
+    debug_assert!(!query.anon());
 
-    ensure_sufficient_stack(|| try_execute_query::<Q, _>(qcx, DUMMY_SP, key, Some(dep_node)));
+    ensure_sufficient_stack(|| try_execute_query(query, qcx, DUMMY_SP, key, Some(dep_node)));
 }
diff --git a/compiler/rustc_resolve/locales/en-US.ftl b/compiler/rustc_resolve/messages.ftl
index 817bb83ed..01f002c94 100644
--- a/compiler/rustc_resolve/locales/en-US.ftl
+++ b/compiler/rustc_resolve/messages.ftl
@@ -42,7 +42,7 @@ resolve_try_adding_local_generic_param_on_method =
     try adding a local generic parameter in this method instead
 
 resolve_help_try_using_local_generic_param =
-    try using a local generic paramter instead
+    try using a local generic parameter instead
 
 resolve_name_is_already_used_as_generic_parameter =
     the name `{$name}` is already used for a generic parameter in this item's generic parameters
@@ -207,5 +207,19 @@ resolve_expected_found =
 resolve_indeterminate =
     cannot determine resolution for the visibility
 
+resolve_tool_module_imported =
+    cannot use a tool module through an import
+    .note = the tool module imported here
+
 resolve_module_only =
     visibility must resolve to a module
+
+resolve_macro_expected_found =
+    expected {$expected}, found {$found} `{$macro_path}`
+
+resolve_remove_surrounding_derive =
+    remove from the surrounding `derive()`
+
+resolve_add_as_non_derive =
+    add as non-Derive macro
+    `#[{$macro_path}]`
diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs
index b1e023f2c..ff0f1f559 100644
--- a/compiler/rustc_resolve/src/build_reduced_graph.rs
+++ b/compiler/rustc_resolve/src/build_reduced_graph.rs
@@ -25,12 +25,9 @@ use rustc_expand::expand::AstFragment;
 use rustc_hir::def::{self, *};
 use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
 use rustc_metadata::creader::LoadedMacro;
-use rustc_middle::bug;
 use rustc_middle::metadata::ModChild;
-use rustc_middle::ty::{self, DefIdTree};
-use rustc_session::cstore::CrateStore;
+use rustc_middle::{bug, ty};
 use rustc_span::hygiene::{ExpnId, LocalExpnId, MacroKind};
-use rustc_span::source_map::respan;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
 
@@ -99,7 +96,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         loop {
             match self.get_module(def_id) {
                 Some(module) => return module,
-                None => def_id = self.parent(def_id),
+                None => def_id = self.tcx.parent(def_id),
             }
         }
     }
@@ -117,35 +114,28 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         }
 
         if !def_id.is_local() {
-            let def_kind = self.cstore().def_kind(def_id);
-            match def_kind {
-                DefKind::Mod | DefKind::Enum | DefKind::Trait => {
-                    let def_key = self.cstore().def_key(def_id);
-                    let parent = def_key.parent.map(|index| {
-                        self.get_nearest_non_block_module(DefId { index, krate: def_id.krate })
-                    });
-                    let name = if let Some(cnum) = def_id.as_crate_root() {
-                        self.cstore().crate_name(cnum)
-                    } else {
-                        def_key.disambiguated_data.data.get_opt_name().expect("module without name")
-                    };
-
-                    let expn_id = self.cstore().module_expansion_untracked(def_id, &self.tcx.sess);
-                    let span = self.cstore().get_span_untracked(def_id, &self.tcx.sess);
-                    Some(self.new_module(
-                        parent,
-                        ModuleKind::Def(def_kind, def_id, name),
-                        expn_id,
-                        span,
-                        // FIXME: Account for `#[no_implicit_prelude]` attributes.
-                        parent.map_or(false, |module| module.no_implicit_prelude),
-                    ))
-                }
-                _ => None,
+            // Query `def_kind` is not used because query system overhead is too expensive here.
+            let def_kind = self.cstore().def_kind_untracked(def_id);
+            if let DefKind::Mod | DefKind::Enum | DefKind::Trait = def_kind {
+                let parent = self
+                    .tcx
+                    .opt_parent(def_id)
+                    .map(|parent_id| self.get_nearest_non_block_module(parent_id));
+                // Query `expn_that_defined` is not used because
+                // hashing spans in its result is expensive.
+                let expn_id = self.cstore().expn_that_defined_untracked(def_id, &self.tcx.sess);
+                return Some(self.new_module(
+                    parent,
+                    ModuleKind::Def(def_kind, def_id, self.tcx.item_name(def_id)),
+                    expn_id,
+                    self.def_span(def_id),
+                    // FIXME: Account for `#[no_implicit_prelude]` attributes.
+                    parent.map_or(false, |module| module.no_implicit_prelude),
+                ));
             }
-        } else {
-            None
         }
+
+        None
     }
 
     pub(crate) fn expn_def_scope(&mut self, expn_id: ExpnId) -> Module<'a> {
@@ -207,6 +197,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
     }
 
     pub(crate) fn build_reduced_graph_external(&mut self, module: Module<'a>) {
+        // Query `module_children` is not used because hashing spans in its result is expensive.
         let children =
             Vec::from_iter(self.cstore().module_children_untracked(module.def_id(), self.tcx.sess));
         for child in children {
@@ -329,13 +320,13 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
         }
     }
 
-    fn insert_field_names_local(&mut self, def_id: DefId, vdata: &ast::VariantData) {
-        let field_names = vdata
-            .fields()
-            .iter()
-            .map(|field| respan(field.span, field.ident.map_or(kw::Empty, |ident| ident.name)))
-            .collect();
-        self.r.field_names.insert(def_id, field_names);
+    fn insert_field_def_ids(&mut self, def_id: LocalDefId, vdata: &ast::VariantData) {
+        if vdata.fields().iter().any(|field| field.is_placeholder) {
+            // The fields are not expanded yet.
+            return;
+        }
+        let def_ids = vdata.fields().iter().map(|field| self.r.local_def_id(field.id).to_def_id());
+        self.r.field_def_ids.insert(def_id, self.r.tcx.arena.alloc_from_iter(def_ids));
     }
 
     fn insert_field_visibilities_local(&mut self, def_id: DefId, vdata: &ast::VariantData) {
@@ -347,12 +338,6 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
         self.r.field_visibility_spans.insert(def_id, field_vis);
     }
 
-    fn insert_field_names_extern(&mut self, def_id: DefId) {
-        let field_names =
-            self.r.cstore().struct_field_names_untracked(def_id, self.r.tcx.sess).collect();
-        self.r.field_names.insert(def_id, field_names);
-    }
-
     fn block_needs_anonymous_module(&mut self, block: &Block) -> bool {
         // If any statements are items, we need to create an anonymous module
         block
@@ -579,7 +564,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             }
             ast::UseTreeKind::Glob => {
                 let kind = ImportKind::Glob {
-                    is_prelude: self.r.tcx.sess.contains_name(&item.attrs, sym::prelude_import),
+                    is_prelude: attr::contains_name(&item.attrs, sym::prelude_import),
                     max_vis: Cell::new(None),
                     id,
                 };
@@ -694,7 +679,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                     expansion.to_expn_id(),
                     item.span,
                     parent.no_implicit_prelude
-                        || self.r.tcx.sess.contains_name(&item.attrs, sym::no_implicit_prelude),
+                        || attr::contains_name(&item.attrs, sym::no_implicit_prelude),
                 );
                 self.r.define(parent, ident, TypeNS, (module, vis, sp, expansion));
 
@@ -703,8 +688,8 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             }
 
             // These items live in the value namespace.
-            ItemKind::Static(_, mt, _) => {
-                let res = Res::Def(DefKind::Static(mt), def_id);
+            ItemKind::Static(box ast::StaticItem { mutability, .. }) => {
+                let res = Res::Def(DefKind::Static(mutability), def_id);
                 self.r.define(parent, ident, ValueNS, (res, vis, sp, expansion));
             }
             ItemKind::Const(..) => {
@@ -750,7 +735,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion));
 
                 // Record field names for error reporting.
-                self.insert_field_names_local(def_id, vdata);
+                self.insert_field_def_ids(local_def_id, vdata);
                 self.insert_field_visibilities_local(def_id, vdata);
 
                 // If this is a tuple or unit struct, define a name
@@ -759,7 +744,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                     // If the structure is marked as non_exhaustive then lower the visibility
                     // to within the crate.
                     let mut ctor_vis = if vis.is_public()
-                        && self.r.tcx.sess.contains_name(&item.attrs, sym::non_exhaustive)
+                        && attr::contains_name(&item.attrs, sym::non_exhaustive)
                     {
                         ty::Visibility::Restricted(CRATE_DEF_ID)
                     } else {
@@ -775,7 +760,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                         let field_vis = self
                             .try_resolve_visibility(&field.vis, false)
                             .unwrap_or(ty::Visibility::Public);
-                        if ctor_vis.is_at_least(field_vis, &*self.r) {
+                        if ctor_vis.is_at_least(field_vis, self.r.tcx) {
                             ctor_vis = field_vis;
                         }
                         ret_fields.push(field_vis.to_def_id());
@@ -790,7 +775,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
 
                     self.r
                         .struct_constructors
-                        .insert(def_id, (ctor_res, ctor_vis.to_def_id(), ret_fields));
+                        .insert(local_def_id, (ctor_res, ctor_vis.to_def_id(), ret_fields));
                 }
             }
 
@@ -799,7 +784,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion));
 
                 // Record field names for error reporting.
-                self.insert_field_names_local(def_id, vdata);
+                self.insert_field_def_ids(local_def_id, vdata);
                 self.insert_field_visibilities_local(def_id, vdata);
             }
 
@@ -946,7 +931,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
     /// Builds the reduced graph for a single item in an external crate.
     fn build_reduced_graph_for_external_crate_res(&mut self, child: ModChild) {
         let parent = self.parent_scope.module;
-        let ModChild { ident, res, vis, span, macro_rules } = child;
+        let ModChild { ident, res, vis, span, .. } = child;
         let res = res.expect_non_local();
         let expansion = self.parent_scope.expansion;
         // Record primary definitions.
@@ -979,9 +964,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 _,
             ) => self.r.define(parent, ident, ValueNS, (res, vis, span, expansion)),
             Res::Def(DefKind::Macro(..), _) | Res::NonMacroAttr(..) => {
-                if !macro_rules {
-                    self.r.define(parent, ident, MacroNS, (res, vis, span, expansion))
-                }
+                self.r.define(parent, ident, MacroNS, (res, vis, span, expansion))
             }
             Res::Def(
                 DefKind::TyParam
@@ -1005,32 +988,6 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             | Res::SelfCtor(..)
             | Res::Err => bug!("unexpected resolution: {:?}", res),
         }
-        // Record some extra data for better diagnostics.
-        match res {
-            Res::Def(DefKind::Struct, def_id) => {
-                let cstore = self.r.cstore();
-                if let Some((ctor_kind, ctor_def_id)) = cstore.ctor_untracked(def_id) {
-                    let ctor_res = Res::Def(DefKind::Ctor(CtorOf::Struct, ctor_kind), ctor_def_id);
-                    let ctor_vis = cstore.visibility_untracked(ctor_def_id);
-                    let field_visibilities =
-                        cstore.struct_field_visibilities_untracked(def_id).collect();
-                    drop(cstore);
-                    self.r
-                        .struct_constructors
-                        .insert(def_id, (ctor_res, ctor_vis, field_visibilities));
-                } else {
-                    drop(cstore);
-                }
-                self.insert_field_names_extern(def_id)
-            }
-            Res::Def(DefKind::Union, def_id) => self.insert_field_names_extern(def_id),
-            Res::Def(DefKind::AssocFn, def_id) => {
-                if self.r.cstore().fn_has_self_parameter_untracked(def_id, self.r.tcx.sess) {
-                    self.r.has_self.insert(def_id);
-                }
-            }
-            _ => {}
-        }
     }
 
     fn add_macro_use_binding(
@@ -1203,12 +1160,11 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
     }
 
     fn proc_macro_stub(&self, item: &ast::Item) -> Option<(MacroKind, Ident, Span)> {
-        if self.r.tcx.sess.contains_name(&item.attrs, sym::proc_macro) {
+        if attr::contains_name(&item.attrs, sym::proc_macro) {
             return Some((MacroKind::Bang, item.ident, item.span));
-        } else if self.r.tcx.sess.contains_name(&item.attrs, sym::proc_macro_attribute) {
+        } else if attr::contains_name(&item.attrs, sym::proc_macro_attribute) {
             return Some((MacroKind::Attr, item.ident, item.span));
-        } else if let Some(attr) = self.r.tcx.sess.find_by_name(&item.attrs, sym::proc_macro_derive)
-        {
+        } else if let Some(attr) = attr::find_by_name(&item.attrs, sym::proc_macro_derive) {
             if let Some(nested_meta) = attr.meta_item_list().and_then(|list| list.get(0).cloned()) {
                 if let Some(ident) = nested_meta.ident() {
                     return Some((MacroKind::Derive, ident, ident.span));
@@ -1263,7 +1219,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
         if macro_rules {
             let ident = ident.normalize_to_macros_2_0();
             self.r.macro_names.insert(ident);
-            let is_macro_export = self.r.tcx.sess.contains_name(&item.attrs, sym::macro_export);
+            let is_macro_export = attr::contains_name(&item.attrs, sym::macro_export);
             let vis = if is_macro_export {
                 ty::Visibility::Public
             } else {
@@ -1414,10 +1370,7 @@ impl<'a, 'b, 'tcx> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b, 'tcx> {
 
         if !(ctxt == AssocCtxt::Impl
             && matches!(item.vis.kind, ast::VisibilityKind::Inherited)
-            && self
-                .r
-                .trait_impl_items
-                .contains(&ty::DefIdTree::local_parent(&*self.r, local_def_id)))
+            && self.r.trait_impl_items.contains(&self.r.tcx.local_parent(local_def_id)))
         {
             // Trait impl item visibility is inherited from its trait when not specified
             // explicitly. In that case we cannot determine it here in early resolve,
@@ -1430,7 +1383,7 @@ impl<'a, 'b, 'tcx> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 AssocItemKind::Const(..) => (DefKind::AssocConst, ValueNS),
                 AssocItemKind::Fn(box Fn { ref sig, .. }) => {
                     if sig.decl.has_self() {
-                        self.r.has_self.insert(def_id);
+                        self.r.has_self.insert(local_def_id);
                     }
                     (DefKind::AssocFn, ValueNS)
                 }
@@ -1526,13 +1479,12 @@ impl<'a, 'b, 'tcx> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b, 'tcx> {
         self.r.visibilities.insert(def_id, vis);
 
         // If the variant is marked as non_exhaustive then lower the visibility to within the crate.
-        let ctor_vis = if vis.is_public()
-            && self.r.tcx.sess.contains_name(&variant.attrs, sym::non_exhaustive)
-        {
-            ty::Visibility::Restricted(CRATE_DEF_ID)
-        } else {
-            vis
-        };
+        let ctor_vis =
+            if vis.is_public() && attr::contains_name(&variant.attrs, sym::non_exhaustive) {
+                ty::Visibility::Restricted(CRATE_DEF_ID)
+            } else {
+                vis
+            };
 
         // Define a constructor name in the value namespace.
         if let Some((ctor_kind, ctor_node_id)) = CtorKind::from_ast(&variant.data) {
@@ -1544,7 +1496,7 @@ impl<'a, 'b, 'tcx> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b, 'tcx> {
         }
 
         // Record field names for error reporting.
-        self.insert_field_names_local(def_id.to_def_id(), &variant.data);
+        self.insert_field_def_ids(def_id, &variant.data);
         self.insert_field_visibilities_local(def_id.to_def_id(), &variant.data);
 
         visit::walk_variant(self, variant);
diff --git a/compiler/rustc_resolve/src/check_unused.rs b/compiler/rustc_resolve/src/check_unused.rs
index b2578e4c4..ae3fd0ede 100644
--- a/compiler/rustc_resolve/src/check_unused.rs
+++ b/compiler/rustc_resolve/src/check_unused.rs
@@ -32,9 +32,10 @@ use rustc_ast::visit::{self, Visitor};
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::{pluralize, MultiSpan};
+use rustc_hir::def::{DefKind, Res};
 use rustc_session::lint::builtin::{MACRO_USE_EXTERN_CRATE, UNUSED_EXTERN_CRATES, UNUSED_IMPORTS};
 use rustc_session::lint::BuiltinLintDiagnostics;
-use rustc_span::symbol::Ident;
+use rustc_span::symbol::{kw, Ident};
 use rustc_span::{Span, DUMMY_SP};
 
 struct UnusedImport<'a> {
@@ -58,6 +59,7 @@ struct UnusedImportCheckVisitor<'a, 'b, 'tcx> {
     base_use_tree: Option<&'a ast::UseTree>,
     base_id: ast::NodeId,
     item_span: Span,
+    base_use_is_pub: bool,
 }
 
 struct ExternCrateToLint {
@@ -110,6 +112,35 @@ impl<'a, 'b, 'tcx> UnusedImportCheckVisitor<'a, 'b, 'tcx> {
             unused: Default::default(),
         })
     }
+
+    fn check_import_as_underscore(&mut self, item: &ast::UseTree, id: ast::NodeId) {
+        match item.kind {
+            ast::UseTreeKind::Simple(Some(ident)) => {
+                if ident.name == kw::Underscore
+                    && !self
+                        .r
+                        .import_res_map
+                        .get(&id)
+                        .map(|per_ns| {
+                            per_ns.iter().filter_map(|res| res.as_ref()).any(|res| {
+                                matches!(res, Res::Def(DefKind::Trait | DefKind::TraitAlias, _))
+                            })
+                        })
+                        .unwrap_or(false)
+                {
+                    self.unused_import(self.base_id).add(id);
+                }
+            }
+            ast::UseTreeKind::Nested(ref items) => self.check_imports_as_underscore(items),
+            _ => {}
+        }
+    }
+
+    fn check_imports_as_underscore(&mut self, items: &[(ast::UseTree, ast::NodeId)]) {
+        for (item, id) in items {
+            self.check_import_as_underscore(item, *id);
+        }
+    }
 }
 
 impl<'a, 'b, 'tcx> Visitor<'a> for UnusedImportCheckVisitor<'a, 'b, 'tcx> {
@@ -119,7 +150,8 @@ impl<'a, 'b, 'tcx> Visitor<'a> for UnusedImportCheckVisitor<'a, 'b, 'tcx> {
             // whether they're used or not. Also ignore imports with a dummy span
             // because this means that they were generated in some fashion by the
             // compiler and we don't need to consider them.
-            ast::ItemKind::Use(..) if item.vis.kind.is_pub() || item.span.is_dummy() => return,
+            ast::ItemKind::Use(..) if item.span.is_dummy() => return,
+            ast::ItemKind::Use(..) => self.base_use_is_pub = item.vis.kind.is_pub(),
             ast::ItemKind::ExternCrate(orig_name) => {
                 self.extern_crate_items.push(ExternCrateToLint {
                     id: item.id,
@@ -146,6 +178,11 @@ impl<'a, 'b, 'tcx> Visitor<'a> for UnusedImportCheckVisitor<'a, 'b, 'tcx> {
             self.base_use_tree = Some(use_tree);
         }
 
+        if self.base_use_is_pub {
+            self.check_import_as_underscore(use_tree, id);
+            return;
+        }
+
         if let ast::UseTreeKind::Nested(ref items) = use_tree.kind {
             if items.is_empty() {
                 self.unused_import(self.base_id).add(id);
@@ -300,6 +337,7 @@ impl Resolver<'_, '_> {
             base_use_tree: None,
             base_id: ast::DUMMY_NODE_ID,
             item_span: DUMMY_SP,
+            base_use_is_pub: false,
         };
         visit::walk_crate(&mut visitor, krate);
 
@@ -355,7 +393,7 @@ impl Resolver<'_, '_> {
             // If we are in the `--test` mode, suppress a help that adds the `#[cfg(test)]`
             // attribute; however, if not, suggest adding the attribute. There is no way to
             // retrieve attributes here because we do not have a `TyCtxt` yet.
-            let test_module_span = if tcx.sess.opts.test {
+            let test_module_span = if tcx.sess.is_test_crate() {
                 None
             } else {
                 let parent_module = visitor.r.get_nearest_non_block_module(
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index e7ff236f8..356d7f365 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -260,9 +260,7 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
                     Async::No => closure_def,
                 }
             }
-            ExprKind::Async(_, async_id, _) => {
-                self.create_def(async_id, DefPathData::ClosureExpr, expr.span)
-            }
+            ExprKind::Async(_, _) => self.create_def(expr.id, DefPathData::ClosureExpr, expr.span),
             _ => self.parent_def,
         };
 
diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs
index 7add59ac6..0c9d30608 100644
--- a/compiler/rustc_resolve/src/diagnostics.rs
+++ b/compiler/rustc_resolve/src/diagnostics.rs
@@ -12,10 +12,10 @@ use rustc_errors::{struct_span_err, SuggestionStyle};
 use rustc_feature::BUILTIN_ATTRIBUTES;
 use rustc_hir::def::Namespace::{self, *};
 use rustc_hir::def::{self, CtorKind, CtorOf, DefKind, NonMacroAttrKind, PerNS};
-use rustc_hir::def_id::{DefId, CRATE_DEF_ID, LOCAL_CRATE};
+use rustc_hir::def_id::{DefId, CRATE_DEF_ID};
 use rustc_hir::PrimTy;
 use rustc_middle::bug;
-use rustc_middle::ty::{DefIdTree, TyCtxt};
+use rustc_middle::ty::TyCtxt;
 use rustc_session::lint::builtin::ABSOLUTE_PATHS_NOT_STARTING_WITH_CRATE;
 use rustc_session::lint::builtin::MACRO_EXPANDED_MACRO_EXPORTS_ACCESSED_BY_ABSOLUTE_PATHS;
 use rustc_session::lint::BuiltinLintDiagnostics;
@@ -555,25 +555,22 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                         return err;
                     }
                     Res::SelfTyAlias { alias_to: def_id, .. } => {
-                        if let Some(impl_span) = self.opt_span(def_id) {
-                            err.span_label(
-                                reduce_impl_span_to_impl_keyword(sm, impl_span),
-                                "`Self` type implicitly declared here, by this `impl`",
-                            );
-                        }
+                        err.span_label(
+                            reduce_impl_span_to_impl_keyword(sm, self.def_span(def_id)),
+                            "`Self` type implicitly declared here, by this `impl`",
+                        );
                         err.span_label(span, "use a type here instead");
                         return err;
                     }
                     Res::Def(DefKind::TyParam, def_id) => {
-                        if let Some(span) = self.opt_span(def_id) {
-                            err.span_label(span, "type parameter from outer function");
-                        }
+                        err.span_label(self.def_span(def_id), "type parameter from outer function");
                         def_id
                     }
                     Res::Def(DefKind::ConstParam, def_id) => {
-                        if let Some(span) = self.opt_span(def_id) {
-                            err.span_label(span, "const parameter from outer function");
-                        }
+                        err.span_label(
+                            self.def_span(def_id),
+                            "const parameter from outer function",
+                        );
                         def_id
                     }
                     _ => {
@@ -589,7 +586,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                     // Try to retrieve the span of the function signature and generate a new
                     // message with a local type or const parameter.
                     let sugg_msg = "try using a local generic parameter instead";
-                    let name = self.opt_name(def_id).unwrap_or(sym::T);
+                    let name = self.tcx.item_name(def_id);
                     let (span, snippet) = if span.is_empty() {
                         let snippet = format!("<{}>", name);
                         (span, snippet)
@@ -1197,7 +1194,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                         segms.push(ast::PathSegment::from_ident(ident));
                         let path = Path { span: name_binding.span, segments: segms, tokens: None };
                         let did = match res {
-                            Res::Def(DefKind::Ctor(..), did) => this.opt_parent(did),
+                            Res::Def(DefKind::Ctor(..), did) => this.tcx.opt_parent(did),
                             _ => res.opt_def_id(),
                         };
 
@@ -1216,15 +1213,11 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                             // a note about editions
                             let note = if let Some(did) = did {
                                 let requires_note = !did.is_local()
-                                    && this.cstore().item_attrs_untracked(did, this.tcx.sess).any(
+                                    && this.tcx.get_attrs(did, sym::rustc_diagnostic_item).any(
                                         |attr| {
-                                            if attr.has_name(sym::rustc_diagnostic_item) {
-                                                [sym::TryInto, sym::TryFrom, sym::FromIterator]
-                                                    .map(|x| Some(x))
-                                                    .contains(&attr.value_str())
-                                            } else {
-                                                false
-                                            }
+                                            [sym::TryInto, sym::TryFrom, sym::FromIterator]
+                                                .map(|x| Some(x))
+                                                .contains(&attr.value_str())
                                         },
                                     );
 
@@ -1373,8 +1366,8 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         }
         if ident.name == kw::Default
             && let ModuleKind::Def(DefKind::Enum, def_id, _) = parent_scope.module.kind
-            && let Some(span) = self.opt_span(def_id)
         {
+            let span = self.def_span(def_id);
             let source_map = self.tcx.sess.source_map();
             let head_span = source_map.guess_head_span(span);
             if let Ok(head) = source_map.span_to_snippet(head_span) {
@@ -1450,11 +1443,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             Some(suggestion) if suggestion.candidate == kw::Underscore => return false,
             Some(suggestion) => suggestion,
         };
-        let def_span = suggestion.res.opt_def_id().and_then(|def_id| match def_id.krate {
-            LOCAL_CRATE => self.opt_span(def_id),
-            _ => Some(self.cstore().get_span_untracked(def_id, self.tcx.sess)),
-        });
-        if let Some(def_span) = def_span {
+        if let Some(def_span) = suggestion.res.opt_def_id().map(|def_id| self.def_span(def_id)) {
             if span.overlaps(def_span) {
                 // Don't suggest typo suggestion for itself like in the following:
                 // error[E0423]: expected function, tuple struct or tuple variant, found struct `X`
@@ -1591,9 +1580,12 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             ctor_def_id,
         )) = binding.kind
         {
-            let def_id = self.parent(ctor_def_id);
-            let fields = self.field_names.get(&def_id)?;
-            return fields.iter().map(|name| name.span).reduce(Span::to); // None for `struct Foo()`
+            let def_id = self.tcx.parent(ctor_def_id);
+            return self
+                .field_def_ids(def_id)?
+                .iter()
+                .map(|&field_id| self.def_span(field_id))
+                .reduce(Span::to); // None for `struct Foo()`
         }
         None
     }
@@ -1615,7 +1607,17 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         let mut err =
             struct_span_err!(self.tcx.sess, ident.span, E0603, "{} `{}` is private", descr, ident);
         err.span_label(ident.span, &format!("private {}", descr));
-        if let Some(span) = ctor_fields_span {
+
+        let mut non_exhaustive = None;
+        // If an ADT is foreign and marked as `non_exhaustive`, then that's
+        // probably why we have the privacy error.
+        // Otherwise, point out if the struct has any private fields.
+        if let Some(def_id) = res.opt_def_id()
+            && !def_id.is_local()
+            && let Some(attr) = self.tcx.get_attr(def_id, sym::non_exhaustive)
+        {
+            non_exhaustive = Some(attr.span);
+        } else if let Some(span) = ctor_fields_span {
             err.span_label(span, "a constructor is private if any of the fields is private");
             if let Res::Def(_, d) = res && let Some(fields) = self.field_visibility_spans.get(&d) {
                 err.multipart_suggestion_verbose(
@@ -1664,6 +1666,14 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             if !first && binding.vis.is_public() {
                 note_span.push_span_label(def_span, "consider importing it directly");
             }
+            // Final step in the import chain, point out if the ADT is `non_exhaustive`
+            // which is probably why this privacy violation occurred.
+            if next_binding.is_none() && let Some(span) = non_exhaustive {
+                note_span.push_span_label(
+                    span,
+                    format!("cannot be constructed because it is `#[non_exhaustive]`"),
+                );
+            }
             err.span_note(note_span, &msg);
         }
 
@@ -1677,8 +1687,8 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
     ) -> Option<Symbol> {
         let mut candidates = self
             .extern_prelude
-            .iter()
-            .map(|(ident, _)| ident.name)
+            .keys()
+            .map(|ident| ident.name)
             .chain(
                 self.module_map
                     .iter()
@@ -2015,7 +2025,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         // 1) some consistent ordering for emitted diagnostics, and
         // 2) `std` suggestions before `core` suggestions.
         let mut extern_crate_names =
-            self.extern_prelude.iter().map(|(ident, _)| ident.name).collect::<Vec<_>>();
+            self.extern_prelude.keys().map(|ident| ident.name).collect::<Vec<_>>();
         extern_crate_names.sort_by(|a, b| b.as_str().partial_cmp(a.as_str()).unwrap());
 
         for name in extern_crate_names.into_iter() {
diff --git a/compiler/rustc_resolve/src/effective_visibilities.rs b/compiler/rustc_resolve/src/effective_visibilities.rs
index 7bd90d7e3..bed579f6b 100644
--- a/compiler/rustc_resolve/src/effective_visibilities.rs
+++ b/compiler/rustc_resolve/src/effective_visibilities.rs
@@ -4,12 +4,13 @@ use rustc_ast::visit;
 use rustc_ast::visit::Visitor;
 use rustc_ast::Crate;
 use rustc_ast::EnumDef;
+use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::intern::Interned;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::def_id::CRATE_DEF_ID;
 use rustc_middle::middle::privacy::Level;
 use rustc_middle::middle::privacy::{EffectiveVisibilities, EffectiveVisibility};
-use rustc_middle::ty::{DefIdTree, Visibility};
+use rustc_middle::ty::Visibility;
 use std::mem;
 
 type ImportId<'a> = Interned<'a, NameBinding<'a>>;
@@ -60,7 +61,7 @@ impl Resolver<'_, '_> {
         // For mod items `nearest_normal_mod` returns its argument, but we actually need its parent.
         let normal_mod_id = self.nearest_normal_mod(def_id);
         if normal_mod_id == def_id {
-            self.opt_local_parent(def_id).map_or(Visibility::Public, Visibility::Restricted)
+            Visibility::Restricted(self.tcx.local_parent(def_id))
         } else {
             Visibility::Restricted(normal_mod_id)
         }
@@ -70,21 +71,20 @@ impl Resolver<'_, '_> {
 impl<'r, 'a, 'tcx> EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
     /// Fills the `Resolver::effective_visibilities` table with public & exported items
     /// For now, this doesn't resolve macros (FIXME) and cannot resolve Impl, as we
-    /// need access to a TyCtxt for that.
+    /// need access to a TyCtxt for that. Returns the set of ambiguous re-exports.
     pub(crate) fn compute_effective_visibilities<'c>(
         r: &'r mut Resolver<'a, 'tcx>,
         krate: &'c Crate,
-    ) {
+    ) -> FxHashSet<Interned<'a, NameBinding<'a>>> {
         let mut visitor = EffectiveVisibilitiesVisitor {
             r,
             def_effective_visibilities: Default::default(),
             import_effective_visibilities: Default::default(),
-            current_private_vis: Visibility::Public,
+            current_private_vis: Visibility::Restricted(CRATE_DEF_ID),
             changed: false,
         };
 
-        visitor.update(CRATE_DEF_ID, CRATE_DEF_ID);
-        visitor.current_private_vis = Visibility::Restricted(CRATE_DEF_ID);
+        visitor.def_effective_visibilities.update_root();
         visitor.set_bindings_effective_visibilities(CRATE_DEF_ID);
 
         while visitor.changed {
@@ -93,18 +93,26 @@ impl<'r, 'a, 'tcx> EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
         }
         visitor.r.effective_visibilities = visitor.def_effective_visibilities;
 
+        let mut exported_ambiguities = FxHashSet::default();
+
         // Update visibilities for import def ids. These are not used during the
         // `EffectiveVisibilitiesVisitor` pass, because we have more detailed binding-based
         // information, but are used by later passes. Effective visibility of an import def id
         // is the maximum value among visibilities of bindings corresponding to that def id.
         for (binding, eff_vis) in visitor.import_effective_visibilities.iter() {
             let NameBindingKind::Import { import, .. } = binding.kind else { unreachable!() };
-            if let Some(node_id) = import.id() {
-                r.effective_visibilities.update_eff_vis(r.local_def_id(node_id), eff_vis, r.tcx)
+            if !binding.is_ambiguity() {
+                if let Some(node_id) = import.id() {
+                    r.effective_visibilities.update_eff_vis(r.local_def_id(node_id), eff_vis, r.tcx)
+                }
+            } else if binding.ambiguity.is_some() && eff_vis.is_public_at_level(Level::Reexported) {
+                exported_ambiguities.insert(*binding);
             }
         }
 
         info!("resolve::effective_visibilities: {:#?}", r.effective_visibilities);
+
+        exported_ambiguities
     }
 
     /// Update effective visibilities of bindings in the given module,
@@ -115,30 +123,38 @@ impl<'r, 'a, 'tcx> EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
         let resolutions = self.r.resolutions(module);
 
         for (_, name_resolution) in resolutions.borrow().iter() {
-            if let Some(mut binding) = name_resolution.borrow().binding() && !binding.is_ambiguity() {
+            if let Some(mut binding) = name_resolution.borrow().binding() {
                 // Set the given effective visibility level to `Level::Direct` and
                 // sets the rest of the `use` chain to `Level::Reexported` until
                 // we hit the actual exported item.
+                //
+                // If the binding is ambiguous, put the root ambiguity binding and all reexports
+                // leading to it into the table. They are used by the `ambiguous_glob_reexports`
+                // lint. For all bindings added to the table this way `is_ambiguity` returns true.
                 let mut parent_id = ParentId::Def(module_id);
                 while let NameBindingKind::Import { binding: nested_binding, .. } = binding.kind {
                     let binding_id = ImportId::new_unchecked(binding);
                     self.update_import(binding_id, parent_id);
 
+                    if binding.ambiguity.is_some() {
+                        // Stop at the root ambiguity, further bindings in the chain should not
+                        // be reexported because the root ambiguity blocks any access to them.
+                        // (Those further bindings are most likely not ambiguities themselves.)
+                        break;
+                    }
+
                     parent_id = ParentId::Import(binding_id);
                     binding = nested_binding;
                 }
 
-                if let Some(def_id) = binding.res().opt_def_id().and_then(|id| id.as_local()) {
+                if binding.ambiguity.is_none()
+                    && let Some(def_id) = binding.res().opt_def_id().and_then(|id| id.as_local()) {
                     self.update_def(def_id, binding.vis.expect_local(), parent_id);
                 }
             }
         }
     }
 
-    fn cheap_private_vis(&self, parent_id: ParentId<'_>) -> Option<Visibility> {
-        matches!(parent_id, ParentId::Def(_)).then_some(self.current_private_vis)
-    }
-
     fn effective_vis_or_private(&mut self, parent_id: ParentId<'a>) -> EffectiveVisibility {
         // Private nodes are only added to the table for caching, they could be added or removed at
         // any moment without consequences, so we don't set `changed` to true when adding them.
@@ -152,15 +168,39 @@ impl<'r, 'a, 'tcx> EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
         }
     }
 
+    /// All effective visibilities for a node are larger or equal than private visibility
+    /// for that node (see `check_invariants` in middle/privacy.rs).
+    /// So if either parent or nominal visibility is the same as private visibility, then
+    /// `min(parent_vis, nominal_vis) <= private_vis`, and the update logic is guaranteed
+    /// to not update anything and we can skip it.
+    ///
+    /// We are checking this condition only if the correct value of private visibility is
+    /// cheaply available, otherwise it does't make sense performance-wise.
+    ///
+    /// `None` is returned if the update can be skipped,
+    /// and cheap private visibility is returned otherwise.
+    fn may_update(
+        &self,
+        nominal_vis: Visibility,
+        parent_id: ParentId<'_>,
+    ) -> Option<Option<Visibility>> {
+        match parent_id {
+            ParentId::Def(def_id) => (nominal_vis != self.current_private_vis
+                && self.r.visibilities[&def_id] != self.current_private_vis)
+                .then_some(Some(self.current_private_vis)),
+            ParentId::Import(_) => Some(None),
+        }
+    }
+
     fn update_import(&mut self, binding: ImportId<'a>, parent_id: ParentId<'a>) {
         let nominal_vis = binding.vis.expect_local();
-        let private_vis = self.cheap_private_vis(parent_id);
+        let Some(cheap_private_vis) = self.may_update(nominal_vis, parent_id) else { return };
         let inherited_eff_vis = self.effective_vis_or_private(parent_id);
         let tcx = self.r.tcx;
         self.changed |= self.import_effective_visibilities.update(
             binding,
             nominal_vis,
-            || private_vis.unwrap_or_else(|| self.r.private_vis_import(binding)),
+            || cheap_private_vis.unwrap_or_else(|| self.r.private_vis_import(binding)),
             inherited_eff_vis,
             parent_id.level(),
             tcx,
@@ -168,20 +208,20 @@ impl<'r, 'a, 'tcx> EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
     }
 
     fn update_def(&mut self, def_id: LocalDefId, nominal_vis: Visibility, parent_id: ParentId<'a>) {
-        let private_vis = self.cheap_private_vis(parent_id);
+        let Some(cheap_private_vis) = self.may_update(nominal_vis, parent_id) else { return };
         let inherited_eff_vis = self.effective_vis_or_private(parent_id);
         let tcx = self.r.tcx;
         self.changed |= self.def_effective_visibilities.update(
             def_id,
             nominal_vis,
-            || private_vis.unwrap_or_else(|| self.r.private_vis_def(def_id)),
+            || cheap_private_vis.unwrap_or_else(|| self.r.private_vis_def(def_id)),
             inherited_eff_vis,
             parent_id.level(),
             tcx,
         );
     }
 
-    fn update(&mut self, def_id: LocalDefId, parent_id: LocalDefId) {
+    fn update_field(&mut self, def_id: LocalDefId, parent_id: LocalDefId) {
         self.update_def(def_id, self.r.visibilities[&def_id], ParentId::Def(parent_id));
     }
 }
@@ -213,14 +253,14 @@ impl<'r, 'ast, 'tcx> Visitor<'ast> for EffectiveVisibilitiesVisitor<'ast, 'r, 't
                 for variant in variants {
                     let variant_def_id = self.r.local_def_id(variant.id);
                     for field in variant.data.fields() {
-                        self.update(self.r.local_def_id(field.id), variant_def_id);
+                        self.update_field(self.r.local_def_id(field.id), variant_def_id);
                     }
                 }
             }
 
             ast::ItemKind::Struct(ref def, _) | ast::ItemKind::Union(ref def, _) => {
                 for field in def.fields() {
-                    self.update(self.r.local_def_id(field.id), def_id);
+                    self.update_field(self.r.local_def_id(field.id), def_id);
                 }
             }
 
diff --git a/compiler/rustc_resolve/src/errors.rs b/compiler/rustc_resolve/src/errors.rs
index 867363f42..afa796cb6 100644
--- a/compiler/rustc_resolve/src/errors.rs
+++ b/compiler/rustc_resolve/src/errors.rs
@@ -470,5 +470,41 @@ pub(crate) struct ExpectedFound {
 pub(crate) struct Indeterminate(#[primary_span] pub(crate) Span);
 
 #[derive(Diagnostic)]
+#[diag(resolve_tool_module_imported)]
+pub(crate) struct ToolModuleImported {
+    #[primary_span]
+    pub(crate) span: Span,
+    #[note]
+    pub(crate) import: Span,
+}
+
+#[derive(Diagnostic)]
 #[diag(resolve_module_only)]
 pub(crate) struct ModuleOnly(#[primary_span] pub(crate) Span);
+
+#[derive(Diagnostic, Default)]
+#[diag(resolve_macro_expected_found)]
+pub(crate) struct MacroExpectedFound<'a> {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) found: &'a str,
+    pub(crate) expected: &'a str,
+    pub(crate) macro_path: &'a str,
+    #[subdiagnostic]
+    pub(crate) remove_surrounding_derive: Option<RemoveSurroundingDerive>,
+    #[subdiagnostic]
+    pub(crate) add_as_non_derive: Option<AddAsNonDerive<'a>>,
+}
+
+#[derive(Subdiagnostic)]
+#[help(resolve_remove_surrounding_derive)]
+pub(crate) struct RemoveSurroundingDerive {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Subdiagnostic)]
+#[help(resolve_add_as_non_derive)]
+pub(crate) struct AddAsNonDerive<'a> {
+    pub(crate) macro_path: &'a str,
+}
diff --git a/compiler/rustc_resolve/src/ident.rs b/compiler/rustc_resolve/src/ident.rs
index 52f0b65fa..5a56d7b99 100644
--- a/compiler/rustc_resolve/src/ident.rs
+++ b/compiler/rustc_resolve/src/ident.rs
@@ -17,7 +17,7 @@ use crate::late::{
     ConstantHasGenerics, ConstantItemKind, HasGenericParams, PathSource, Rib, RibKind,
 };
 use crate::macros::{sub_namespace_match, MacroRulesScope};
-use crate::{AmbiguityError, AmbiguityErrorMisc, AmbiguityKind, Determinacy, Finalize};
+use crate::{errors, AmbiguityError, AmbiguityErrorMisc, AmbiguityKind, Determinacy, Finalize};
 use crate::{Import, ImportKind, LexicalScopeBinding, Module, ModuleKind, ModuleOrUniformRoot};
 use crate::{NameBinding, NameBindingKind, ParentScope, PathResult, PrivacyError, Res};
 use crate::{ResolutionError, Resolver, Scope, ScopeSet, Segment, ToNameBinding, Weak};
@@ -389,7 +389,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             }
         }
 
-        assert!(force || !finalize.is_some()); // `finalize` implies `force`
+        assert!(force || finalize.is_none()); // `finalize` implies `force`
 
         // Make sure `self`, `super` etc produce an error when passed to here.
         if orig_ident.is_path_segment_keyword() {
@@ -869,17 +869,19 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         let resolution =
             self.resolution(module, key).try_borrow_mut().map_err(|_| (Determined, Weak::No))?; // This happens when there is a cycle of imports.
 
-        if let Some(Finalize { path_span, report_private, .. }) = finalize {
-            // If the primary binding is unusable, search further and return the shadowed glob
-            // binding if it exists. What we really want here is having two separate scopes in
-            // a module - one for non-globs and one for globs, but until that's done use this
-            // hack to avoid inconsistent resolution ICEs during import validation.
-            let binding = [resolution.binding, resolution.shadowed_glob].into_iter().find_map(
-                |binding| match (binding, ignore_binding) {
+        // If the primary binding is unusable, search further and return the shadowed glob
+        // binding if it exists. What we really want here is having two separate scopes in
+        // a module - one for non-globs and one for globs, but until that's done use this
+        // hack to avoid inconsistent resolution ICEs during import validation.
+        let binding =
+            [resolution.binding, resolution.shadowed_glob].into_iter().find_map(|binding| {
+                match (binding, ignore_binding) {
                     (Some(binding), Some(ignored)) if ptr::eq(binding, ignored) => None,
                     _ => binding,
-                },
-            );
+                }
+            });
+
+        if let Some(Finalize { path_span, report_private, .. }) = finalize {
             let Some(binding) = binding else {
                 return Err((Determined, Weak::No));
             };
@@ -927,15 +929,12 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         }
 
         let check_usable = |this: &mut Self, binding: &'a NameBinding<'a>| {
-            if let Some(ignored) = ignore_binding && ptr::eq(binding, ignored) {
-                return Err((Determined, Weak::No));
-            }
             let usable = this.is_accessible_from(binding.vis, parent_scope.module);
             if usable { Ok(binding) } else { Err((Determined, Weak::No)) }
         };
 
         // Items and single imports are not shadowable, if we have one, then it's determined.
-        if let Some(binding) = resolution.binding {
+        if let Some(binding) = binding {
             if !binding.is_glob_import() {
                 return check_usable(self, binding);
             }
@@ -952,6 +951,14 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             if !self.is_accessible_from(import_vis, parent_scope.module) {
                 continue;
             }
+            if let Some(ignored) = ignore_binding &&
+                let NameBindingKind::Import { import, .. } = ignored.kind &&
+                ptr::eq(import, &**single_import) {
+                // Ignore not just the binding itself, but if it has a shadowed_glob,
+                // ignore that, too, because this loop is supposed to only process
+                // named imports.
+                continue;
+            }
             let Some(module) = single_import.imported_module.get() else {
                 return Err((Undetermined, Weak::No));
             };
@@ -989,7 +996,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         // and prohibit access to macro-expanded `macro_export` macros instead (unless restricted
         // shadowing is enabled, see `macro_expanded_macro_export_errors`).
         let unexpanded_macros = !module.unexpanded_invocations.borrow().is_empty();
-        if let Some(binding) = resolution.binding {
+        if let Some(binding) = binding {
             if !unexpanded_macros || ns == MacroNS || restricted_shadowing {
                 return check_usable(self, binding);
             } else {
@@ -1357,7 +1364,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                 }
             };
 
-            let is_last = i == path.len() - 1;
+            let is_last = i + 1 == path.len();
             let ns = if is_last { opt_ns.unwrap_or(TypeNS) } else { TypeNS };
             let name = ident.name;
 
@@ -1494,16 +1501,12 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                     if let Some(next_module) = binding.module() {
                         module = Some(ModuleOrUniformRoot::Module(next_module));
                         record_segment_res(self, res);
-                    } else if res == Res::ToolMod && i + 1 != path.len() {
+                    } else if res == Res::ToolMod && !is_last && opt_ns.is_some() {
                         if binding.is_import() {
-                            self.tcx
-                                .sess
-                                .struct_span_err(
-                                    ident.span,
-                                    "cannot use a tool module through an import",
-                                )
-                                .span_note(binding.span, "the tool module imported here")
-                                .emit();
+                            self.tcx.sess.emit_err(errors::ToolModuleImported {
+                                span: ident.span,
+                                import: binding.span,
+                            });
                         }
                         let res = Res::NonMacroAttr(NonMacroAttrKind::Tool);
                         return PathResult::NonModule(PartialRes::new(res));
diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs
index 4dab0836d..3c22d51c3 100644
--- a/compiler/rustc_resolve/src/imports.rs
+++ b/compiler/rustc_resolve/src/imports.rs
@@ -17,14 +17,18 @@ use rustc_data_structures::intern::Interned;
 use rustc_errors::{pluralize, struct_span_err, Applicability, MultiSpan};
 use rustc_hir::def::{self, DefKind, PartialRes};
 use rustc_middle::metadata::ModChild;
+use rustc_middle::metadata::Reexport;
 use rustc_middle::span_bug;
 use rustc_middle::ty;
-use rustc_session::lint::builtin::{PUB_USE_OF_PRIVATE_EXTERN_CRATE, UNUSED_IMPORTS};
+use rustc_session::lint::builtin::{
+    AMBIGUOUS_GLOB_REEXPORTS, PUB_USE_OF_PRIVATE_EXTERN_CRATE, UNUSED_IMPORTS,
+};
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::hygiene::LocalExpnId;
 use rustc_span::symbol::{kw, Ident, Symbol};
 use rustc_span::Span;
+use smallvec::SmallVec;
 
 use std::cell::Cell;
 use std::{mem, ptr};
@@ -85,20 +89,28 @@ impl<'a> std::fmt::Debug for ImportKind<'a> {
             Single {
                 ref source,
                 ref target,
+                ref source_bindings,
+                ref target_bindings,
                 ref type_ns_only,
                 ref nested,
                 ref id,
-                // Ignore the following to avoid an infinite loop while printing.
-                source_bindings: _,
-                target_bindings: _,
             } => f
                 .debug_struct("Single")
                 .field("source", source)
                 .field("target", target)
+                // Ignore the nested bindings to avoid an infinite loop while printing.
+                .field(
+                    "source_bindings",
+                    &source_bindings.clone().map(|b| b.into_inner().map(|_| format_args!(".."))),
+                )
+                .field(
+                    "target_bindings",
+                    &target_bindings.clone().map(|b| b.into_inner().map(|_| format_args!(".."))),
+                )
                 .field("type_ns_only", type_ns_only)
                 .field("nested", nested)
                 .field("id", id)
-                .finish_non_exhaustive(),
+                .finish(),
             Glob { ref is_prelude, ref max_vis, ref id } => f
                 .debug_struct("Glob")
                 .field("is_prelude", is_prelude)
@@ -180,6 +192,17 @@ impl<'a> Import<'a> {
             ImportKind::MacroUse | ImportKind::MacroExport => None,
         }
     }
+
+    fn simplify(&self, r: &Resolver<'_, '_>) -> Reexport {
+        let to_def_id = |id| r.local_def_id(id).to_def_id();
+        match self.kind {
+            ImportKind::Single { id, .. } => Reexport::Single(to_def_id(id)),
+            ImportKind::Glob { id, .. } => Reexport::Glob(to_def_id(id)),
+            ImportKind::ExternCrate { id, .. } => Reexport::ExternCrate(to_def_id(id)),
+            ImportKind::MacroUse => Reexport::MacroUse,
+            ImportKind::MacroExport => Reexport::MacroExport,
+        }
+    }
 }
 
 /// Records information about the resolution of a name in a namespace of a module.
@@ -245,7 +268,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         import: &'a Import<'a>,
     ) -> &'a NameBinding<'a> {
         let import_vis = import.expect_vis().to_def_id();
-        let vis = if binding.vis.is_at_least(import_vis, self)
+        let vis = if binding.vis.is_at_least(import_vis, self.tcx)
             || pub_use_of_private_extern_crate_hack(import, binding)
         {
             import_vis
@@ -255,7 +278,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
 
         if let ImportKind::Glob { ref max_vis, .. } = import.kind {
             if vis == import_vis
-                || max_vis.get().map_or(true, |max_vis| vis.is_at_least(max_vis, self))
+                || max_vis.get().map_or(true, |max_vis| vis.is_at_least(max_vis, self.tcx))
             {
                 max_vis.set(Some(vis.expect_local()))
             }
@@ -294,7 +317,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                                 old_binding,
                                 binding,
                             ));
-                        } else if !old_binding.vis.is_at_least(binding.vis, &*this) {
+                        } else if !old_binding.vis.is_at_least(binding.vis, this.tcx) {
                             // We are glob-importing the same item but with greater visibility.
                             resolution.binding = Some(binding);
                         }
@@ -415,13 +438,17 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
     /// Resolves all imports for the crate. This method performs the fixed-
     /// point iteration.
     pub(crate) fn resolve_imports(&mut self) {
-        let mut prev_num_indeterminates = self.indeterminate_imports.len() + 1;
-        while self.indeterminate_imports.len() < prev_num_indeterminates {
-            prev_num_indeterminates = self.indeterminate_imports.len();
+        let mut prev_indeterminate_count = usize::MAX;
+        let mut indeterminate_count = self.indeterminate_imports.len() * 3;
+        while indeterminate_count < prev_indeterminate_count {
+            prev_indeterminate_count = indeterminate_count;
+            indeterminate_count = 0;
             for import in mem::take(&mut self.indeterminate_imports) {
-                match self.resolve_import(&import) {
-                    true => self.determined_imports.push(import),
-                    false => self.indeterminate_imports.push(import),
+                let import_indeterminate_count = self.resolve_import(&import);
+                indeterminate_count += import_indeterminate_count;
+                match import_indeterminate_count {
+                    0 => self.determined_imports.push(import),
+                    _ => self.indeterminate_imports.push(import),
                 }
             }
         }
@@ -498,6 +525,34 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         }
     }
 
+    pub(crate) fn check_reexport_ambiguities(
+        &mut self,
+        exported_ambiguities: FxHashSet<Interned<'a, NameBinding<'a>>>,
+    ) {
+        for module in self.arenas.local_modules().iter() {
+            module.for_each_child(self, |this, ident, ns, binding| {
+                if let NameBindingKind::Import { import, .. } = binding.kind
+                && let Some((amb_binding, _)) = binding.ambiguity
+                && binding.res() != Res::Err
+                && exported_ambiguities.contains(&Interned::new_unchecked(binding))
+                {
+                    this.lint_buffer.buffer_lint_with_diagnostic(
+                        AMBIGUOUS_GLOB_REEXPORTS,
+                        import.root_id,
+                        import.root_span,
+                        "ambiguous glob re-exports",
+                        BuiltinLintDiagnostics::AmbiguousGlobReexports {
+                            name: ident.to_string(),
+                            namespace: ns.descr().to_string(),
+                            first_reexport_span: import.root_span,
+                            duplicate_reexport_span: amb_binding.span,
+                        },
+                    );
+                }
+            });
+        }
+    }
+
     fn throw_unresolved_import_error(&self, errors: Vec<(&Import<'_>, UnresolvedImportError)>) {
         if errors.is_empty() {
             return;
@@ -573,9 +628,13 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         diag.emit();
     }
 
-    /// Attempts to resolve the given import, returning true if its resolution is determined.
-    /// If successful, the resolved bindings are written into the module.
-    fn resolve_import(&mut self, import: &'a Import<'a>) -> bool {
+    /// Attempts to resolve the given import, returning:
+    /// - `0` means its resolution is determined.
+    /// - Other values mean that indeterminate exists under certain namespaces.
+    ///
+    /// Meanwhile, if resolve successful, the resolved bindings are written
+    /// into the module.
+    fn resolve_import(&mut self, import: &'a Import<'a>) -> usize {
         debug!(
             "(resolving import for module) resolving import `{}::...` in `{}`",
             Segment::names_to_string(&import.module_path),
@@ -593,8 +652,8 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
 
             match path_res {
                 PathResult::Module(module) => module,
-                PathResult::Indeterminate => return false,
-                PathResult::NonModule(..) | PathResult::Failed { .. } => return true,
+                PathResult::Indeterminate => return 3,
+                PathResult::NonModule(..) | PathResult::Failed { .. } => return 0,
             }
         };
 
@@ -610,12 +669,12 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             } => (source, target, source_bindings, target_bindings, type_ns_only),
             ImportKind::Glob { .. } => {
                 self.resolve_glob_import(import);
-                return true;
+                return 0;
             }
             _ => unreachable!(),
         };
 
-        let mut indeterminate = false;
+        let mut indeterminate_count = 0;
         self.per_ns(|this, ns| {
             if !type_ns_only || ns == TypeNS {
                 if let Err(Undetermined) = source_bindings[ns].get() {
@@ -638,7 +697,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
 
                 let parent = import.parent_scope.module;
                 match source_bindings[ns].get() {
-                    Err(Undetermined) => indeterminate = true,
+                    Err(Undetermined) => indeterminate_count += 1,
                     // Don't update the resolution, because it was never added.
                     Err(Determined) if target.name == kw::Underscore => {}
                     Ok(binding) if binding.is_importable() => {
@@ -662,7 +721,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             }
         });
 
-        !indeterminate
+        indeterminate_count
     }
 
     /// Performs final import resolution, consistency checks and error reporting.
@@ -786,7 +845,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                     }
                     if !is_prelude
                     && let Some(max_vis) = max_vis.get()
-                    && !max_vis.is_at_least(import.expect_vis(), &*self)
+                    && !max_vis.is_at_least(import.expect_vis(), self.tcx)
                 {
                     let msg = "glob import doesn't reexport anything because no candidate is public enough";
                     self.lint_buffer.buffer_lint(UNUSED_IMPORTS, id, import.span, msg);
@@ -977,7 +1036,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         let mut crate_private_reexport = false;
         self.per_ns(|this, ns| {
             if let Ok(binding) = source_bindings[ns].get() {
-                if !binding.vis.is_at_least(import.expect_vis(), &*this) {
+                if !binding.vis.is_at_least(import.expect_vis(), this.tcx) {
                     reexport_error = Some((ns, binding));
                     if let ty::Visibility::Restricted(binding_def_id) = binding.vis {
                         if binding_def_id.is_top_level_module() {
@@ -1202,24 +1261,38 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         *module.globs.borrow_mut() = Vec::new();
 
         if let Some(def_id) = module.opt_def_id() {
+            let mut non_reexports = Vec::new();
             let mut reexports = Vec::new();
 
             module.for_each_child(self, |this, ident, _, binding| {
-                if let Some(res) = this.is_reexport(binding) {
+                let res = binding.res().expect_non_local();
+                if !binding.is_import() {
+                    non_reexports.push(res.def_id().expect_local());
+                } else if res != def::Res::Err && !binding.is_ambiguity() {
+                    let mut reexport_chain = SmallVec::new();
+                    let mut next_binding = binding;
+                    while let NameBindingKind::Import { binding, import, .. } = next_binding.kind {
+                        reexport_chain.push(import.simplify(this));
+                        next_binding = binding;
+                    }
+
                     reexports.push(ModChild {
                         ident,
                         res,
                         vis: binding.vis,
                         span: binding.span,
-                        macro_rules: false,
+                        reexport_chain,
                     });
                 }
             });
 
+            // Should be fine because this code is only called for local modules.
+            let def_id = def_id.expect_local();
+            if !non_reexports.is_empty() {
+                self.module_children_non_reexports.insert(def_id, non_reexports);
+            }
             if !reexports.is_empty() {
-                // Call to `expect_local` should be fine because current
-                // code is only called for local modules.
-                self.reexport_map.insert(def_id.expect_local(), reexports);
+                self.module_children_reexports.insert(def_id, reexports);
             }
         }
     }
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 7df17376b..90a2fa89c 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -22,7 +22,6 @@ use rustc_hir::def::{self, CtorKind, DefKind, LifetimeRes, PartialRes, PerNS};
 use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::{BindingAnnotation, PrimTy, TraitCandidate};
 use rustc_middle::middle::resolve_bound_vars::Set1;
-use rustc_middle::ty::DefIdTree;
 use rustc_middle::{bug, span_bug};
 use rustc_session::config::{CrateType, ResolveDocLinks};
 use rustc_session::lint;
@@ -123,6 +122,12 @@ pub(crate) enum ConstantItemKind {
     Static,
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+enum RecordPartialRes {
+    Yes,
+    No,
+}
+
 /// The rib kind restricts certain accesses,
 /// e.g. to a `Res::Local` of an outer item.
 #[derive(Copy, Clone, Debug)]
@@ -591,10 +596,9 @@ struct LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
     parent_scope: ParentScope<'a>,
 
     /// The current set of local scopes for types and values.
-    /// FIXME #4948: Reuse ribs to avoid allocation.
     ribs: PerNS<Vec<Rib<'a>>>,
 
-    /// Previous poped `rib`, only used for diagnostic.
+    /// Previous popped `rib`, only used for diagnostic.
     last_block_rib: Option<Rib<'a>>,
 
     /// The current set of local scopes, for labels.
@@ -1219,7 +1223,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             lifetime_ribs: Vec::new(),
             lifetime_elision_candidates: None,
             current_trait_ref: None,
-            diagnostic_metadata: Box::new(DiagnosticMetadata::default()),
+            diagnostic_metadata: Default::default(),
             // errors at module scope should always be reported
             in_func_body: false,
             lifetime_uses: Default::default(),
@@ -1479,8 +1483,9 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                                     } else {
                                         LifetimeUseSet::Many
                                     }),
-                                    LifetimeRibKind::Generics { .. } => None,
-                                    LifetimeRibKind::ConstGeneric | LifetimeRibKind::AnonConst => {
+                                    LifetimeRibKind::Generics { .. }
+                                    | LifetimeRibKind::ConstGeneric => None,
+                                    LifetimeRibKind::AnonConst => {
                                         span_bug!(ident.span, "unexpected rib kind: {:?}", rib.kind)
                                     }
                                 })
@@ -1671,8 +1676,12 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             // Figure out if this is a type/trait segment,
             // which may need lifetime elision performed.
             let type_def_id = match partial_res.base_res() {
-                Res::Def(DefKind::AssocTy, def_id) if i + 2 == proj_start => self.r.parent(def_id),
-                Res::Def(DefKind::Variant, def_id) if i + 1 == proj_start => self.r.parent(def_id),
+                Res::Def(DefKind::AssocTy, def_id) if i + 2 == proj_start => {
+                    self.r.tcx.parent(def_id)
+                }
+                Res::Def(DefKind::Variant, def_id) if i + 1 == proj_start => {
+                    self.r.tcx.parent(def_id)
+                }
                 Res::Def(DefKind::Struct, def_id)
                 | Res::Def(DefKind::Union, def_id)
                 | Res::Def(DefKind::Enum, def_id)
@@ -2342,7 +2351,8 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                 });
             }
 
-            ItemKind::Static(ref ty, _, ref expr) | ItemKind::Const(_, ref ty, ref expr) => {
+            ItemKind::Static(box ast::StaticItem { ref ty, ref expr, .. })
+            | ItemKind::Const(box ast::ConstItem { ref ty, ref expr, .. }) => {
                 self.with_static_rib(|this| {
                     this.with_lifetime_rib(LifetimeRibKind::Elided(LifetimeRes::Static), |this| {
                         this.visit_ty(ty);
@@ -2417,8 +2427,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                     .iter()
                     .rfind(|r| matches!(r.kind, ItemRibKind(_)))
                     .expect("associated item outside of an item");
-                seen_bindings
-                    .extend(parent_rib.bindings.iter().map(|(ident, _)| (*ident, ident.span)));
+                seen_bindings.extend(parent_rib.bindings.keys().map(|ident| (*ident, ident.span)));
             };
             add_bindings_for_ns(ValueNS);
             add_bindings_for_ns(TypeNS);
@@ -2621,11 +2630,11 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
         for item in trait_items {
             self.resolve_doc_links(&item.attrs, MaybeExported::Ok(item.id));
             match &item.kind {
-                AssocItemKind::Const(_, ty, default) => {
+                AssocItemKind::Const(box ast::ConstItem { ty, expr, .. }) => {
                     self.visit_ty(ty);
                     // Only impose the restrictions of `ConstRibKind` for an
                     // actual constant expression in a provided default.
-                    if let Some(expr) = default {
+                    if let Some(expr) = expr {
                         // We allow arbitrary const expressions inside of associated consts,
                         // even if they are potentially not const evaluatable.
                         //
@@ -2678,6 +2687,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                 &path,
                 PathSource::Trait(AliasPossibility::No),
                 Finalize::new(trait_ref.ref_id, trait_ref.path.span),
+                RecordPartialRes::Yes,
             );
             self.diagnostic_metadata.currently_processing_impl_trait = None;
             if let Some(def_id) = res.expect_full_res().opt_def_id() {
@@ -2796,7 +2806,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
         use crate::ResolutionError::*;
         self.resolve_doc_links(&item.attrs, MaybeExported::ImplItem(trait_id.ok_or(&item.vis)));
         match &item.kind {
-            AssocItemKind::Const(_, ty, default) => {
+            AssocItemKind::Const(box ast::ConstItem { ty, expr, .. }) => {
                 debug!("resolve_implementation AssocItemKind::Const");
                 // If this is a trait impl, ensure the const
                 // exists in trait
@@ -2811,7 +2821,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                 );
 
                 self.visit_ty(ty);
-                if let Some(expr) = default {
+                if let Some(expr) = expr {
                     // We allow arbitrary const expressions inside of associated consts,
                     // even if they are potentially not const evaluatable.
                     //
@@ -3373,7 +3383,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                         participle: "defined",
                         article: res.article(),
                         shadowed_binding: res,
-                        shadowed_binding_span: self.r.opt_span(def_id).expect("const parameter defined outside of local crate"),
+                        shadowed_binding_span: self.r.def_span(def_id),
                     }
                 );
                 None
@@ -3416,6 +3426,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             &Segment::from_path(path),
             source,
             Finalize::new(id, path.span),
+            RecordPartialRes::Yes,
         );
     }
 
@@ -3426,6 +3437,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
         path: &[Segment],
         source: PathSource<'ast>,
         finalize: Finalize,
+        record_partial_res: RecordPartialRes,
     ) -> PartialRes {
         let ns = source.namespace();
 
@@ -3454,8 +3466,8 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                         sugg.to_string(),
                         Applicability::MaybeIncorrect,
                     ))
-                } else if res.is_none() && matches!(source, PathSource::Type) {
-                    this.report_missing_type_error(path)
+                } else if res.is_none() && let PathSource::Type | PathSource::Expr(_) = source {
+                    this.suggest_adding_generic_parameter(path, source)
                 } else {
                     None
                 };
@@ -3632,7 +3644,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             _ => report_errors(self, None),
         };
 
-        if !matches!(source, PathSource::TraitItem(..)) {
+        if record_partial_res == RecordPartialRes::Yes {
             // Avoid recording definition of `A::B` in `<T as A>::B::C`.
             self.r.record_partial_res(node_id, partial_res);
             self.resolve_elided_lifetimes_in_path(node_id, partial_res, path, source, path_span);
@@ -3736,7 +3748,25 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                 )));
             }
 
-            // Make sure `A::B` in `<T as A::B>::C` is a trait item.
+            let num_privacy_errors = self.r.privacy_errors.len();
+            // Make sure that `A` in `<T as A>::B::C` is a trait.
+            let trait_res = self.smart_resolve_path_fragment(
+                &None,
+                &path[..qself.position],
+                PathSource::Trait(AliasPossibility::No),
+                Finalize::new(finalize.node_id, qself.path_span),
+                RecordPartialRes::No,
+            );
+
+            if trait_res.expect_full_res() == Res::Err {
+                return Ok(Some(trait_res));
+            }
+
+            // Truncate additional privacy errors reported above,
+            // because they'll be recomputed below.
+            self.r.privacy_errors.truncate(num_privacy_errors);
+
+            // Make sure `A::B` in `<T as A>::B::C` is a trait item.
             //
             // Currently, `path` names the full item (`A::B::C`, in
             // our example). so we extract the prefix of that that is
@@ -3749,6 +3779,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                 &path[..=qself.position],
                 PathSource::TraitItem(ns),
                 Finalize::with_root_span(finalize.node_id, finalize.path_span, qself.path_span),
+                RecordPartialRes::No,
             );
 
             // The remaining segments (the `C` in our example) will
@@ -4233,7 +4264,10 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             {
                 return;
             }
-            ResolveDocLinks::Exported if !maybe_exported.eval(self.r) => {
+            ResolveDocLinks::Exported
+                if !maybe_exported.eval(self.r)
+                    && !rustdoc::has_primitive_or_keyword_docs(attrs) =>
+            {
                 return;
             }
             ResolveDocLinks::ExportedMetadata
diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs
index b8ddc4552..37fbfad2d 100644
--- a/compiler/rustc_resolve/src/late/diagnostics.rs
+++ b/compiler/rustc_resolve/src/late/diagnostics.rs
@@ -19,9 +19,8 @@ use rustc_errors::{
 use rustc_hir as hir;
 use rustc_hir::def::Namespace::{self, *};
 use rustc_hir::def::{self, CtorKind, CtorOf, DefKind};
-use rustc_hir::def_id::{DefId, CRATE_DEF_ID, LOCAL_CRATE};
+use rustc_hir::def_id::{DefId, CRATE_DEF_ID};
 use rustc_hir::PrimTy;
-use rustc_middle::ty::DefIdTree;
 use rustc_session::lint;
 use rustc_session::parse::feature_err;
 use rustc_session::Session;
@@ -167,13 +166,6 @@ impl TypoCandidate {
 }
 
 impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
-    fn def_span(&self, def_id: DefId) -> Option<Span> {
-        match def_id.krate {
-            LOCAL_CRATE => self.r.opt_span(def_id),
-            _ => Some(self.r.cstore().get_span_untracked(def_id, self.r.tcx.sess)),
-        }
-    }
-
     fn make_base_error(
         &mut self,
         path: &[Segment],
@@ -192,7 +184,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 span,
                 span_label: match res {
                     Res::Def(kind, def_id) if kind == DefKind::TyParam => {
-                        self.def_span(def_id).map(|span| (span, "found this type parameter"))
+                        Some((self.r.def_span(def_id), "found this type parameter"))
                     }
                     _ => None,
                 },
@@ -1296,28 +1288,46 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 }
                 PathSource::Expr(_) | PathSource::TupleStruct(..) | PathSource::Pat => {
                     let span = find_span(&source, err);
-                    if let Some(span) = self.def_span(def_id) {
-                        err.span_label(span, &format!("`{}` defined here", path_str));
-                    }
-                    let (tail, descr, applicability) = match source {
-                        PathSource::Pat | PathSource::TupleStruct(..) => {
-                            ("", "pattern", Applicability::MachineApplicable)
-                        }
-                        _ => (": val", "literal", Applicability::HasPlaceholders),
-                    };
-                    let (fields, applicability) = match self.r.field_names.get(&def_id) {
-                        Some(fields) => (
-                            fields
-                                .iter()
-                                .map(|f| format!("{}{}", f.node, tail))
-                                .collect::<Vec<String>>()
-                                .join(", "),
-                            applicability,
+                    err.span_label(self.r.def_span(def_id), &format!("`{path_str}` defined here"));
+
+                    let (tail, descr, applicability, old_fields) = match source {
+                        PathSource::Pat => ("", "pattern", Applicability::MachineApplicable, None),
+                        PathSource::TupleStruct(_, args) => (
+                            "",
+                            "pattern",
+                            Applicability::MachineApplicable,
+                            Some(
+                                args.iter()
+                                    .map(|a| self.r.tcx.sess.source_map().span_to_snippet(*a).ok())
+                                    .collect::<Vec<Option<String>>>(),
+                            ),
                         ),
+                        _ => (": val", "literal", Applicability::HasPlaceholders, None),
+                    };
+                    let field_ids = self.r.field_def_ids(def_id);
+                    let (fields, applicability) = match field_ids {
+                        Some(field_ids) => {
+                            let fields = field_ids.iter().map(|&id| self.r.tcx.item_name(id));
+
+                            let fields = if let Some(old_fields) = old_fields {
+                                fields
+                                    .enumerate()
+                                    .map(|(idx, new)| (new, old_fields.get(idx)))
+                                    .map(|(new, old)| {
+                                        let new = new.to_ident_string();
+                                        if let Some(Some(old)) = old && new != *old { format!("{}: {}", new, old) } else { new }
+                                    })
+                                    .collect::<Vec<String>>()
+                            } else {
+                                fields.map(|f| format!("{f}{tail}")).collect::<Vec<String>>()
+                            };
+
+                            (fields.join(", "), applicability)
+                        }
                         None => ("/* fields */".to_string(), Applicability::HasPlaceholders),
                     };
-                    let pad = match self.r.field_names.get(&def_id) {
-                        Some(fields) if fields.is_empty() => "",
+                    let pad = match field_ids {
+                        Some(field_ids) if field_ids.is_empty() => "",
                         _ => " ",
                     };
                     err.span_suggestion(
@@ -1360,17 +1370,14 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 if self.r.tcx.sess.is_nightly_build() {
                     let msg = "you might have meant to use `#![feature(trait_alias)]` instead of a \
                                `type` alias";
-                    if let Some(span) = self.def_span(def_id) {
-                        if let Ok(snip) = self.r.tcx.sess.source_map().span_to_snippet(span) {
-                            // The span contains a type alias so we should be able to
-                            // replace `type` with `trait`.
-                            let snip = snip.replacen("type", "trait", 1);
-                            err.span_suggestion(span, msg, snip, Applicability::MaybeIncorrect);
-                        } else {
-                            err.span_help(span, msg);
-                        }
+                    let span = self.r.def_span(def_id);
+                    if let Ok(snip) = self.r.tcx.sess.source_map().span_to_snippet(span) {
+                        // The span contains a type alias so we should be able to
+                        // replace `type` with `trait`.
+                        let snip = snip.replacen("type", "trait", 1);
+                        err.span_suggestion(span, msg, snip, Applicability::MaybeIncorrect);
                     } else {
-                        err.help(msg);
+                        err.span_help(span, msg);
                     }
                 }
             }
@@ -1409,19 +1416,38 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 self.suggest_using_enum_variant(err, source, def_id, span);
             }
             (Res::Def(DefKind::Struct, def_id), source) if ns == ValueNS => {
-                let (ctor_def, ctor_vis, fields) =
-                    if let Some(struct_ctor) = self.r.struct_constructors.get(&def_id).cloned() {
-                        if let PathSource::Expr(Some(parent)) = source {
-                            if let ExprKind::Field(..) | ExprKind::MethodCall(..) = parent.kind {
-                                bad_struct_syntax_suggestion(def_id);
-                                return true;
-                            }
+                let struct_ctor = match def_id.as_local() {
+                    Some(def_id) => self.r.struct_constructors.get(&def_id).cloned(),
+                    None => {
+                        let ctor = self.r.cstore().ctor_untracked(def_id);
+                        ctor.map(|(ctor_kind, ctor_def_id)| {
+                            let ctor_res =
+                                Res::Def(DefKind::Ctor(CtorOf::Struct, ctor_kind), ctor_def_id);
+                            let ctor_vis = self.r.tcx.visibility(ctor_def_id);
+                            let field_visibilities = self
+                                .r
+                                .tcx
+                                .associated_item_def_ids(def_id)
+                                .iter()
+                                .map(|field_id| self.r.tcx.visibility(field_id))
+                                .collect();
+                            (ctor_res, ctor_vis, field_visibilities)
+                        })
+                    }
+                };
+
+                let (ctor_def, ctor_vis, fields) = if let Some(struct_ctor) = struct_ctor {
+                    if let PathSource::Expr(Some(parent)) = source {
+                        if let ExprKind::Field(..) | ExprKind::MethodCall(..) = parent.kind {
+                            bad_struct_syntax_suggestion(def_id);
+                            return true;
                         }
-                        struct_ctor
-                    } else {
-                        bad_struct_syntax_suggestion(def_id);
-                        return true;
-                    };
+                    }
+                    struct_ctor
+                } else {
+                    bad_struct_syntax_suggestion(def_id);
+                    return true;
+                };
 
                 let is_accessible = self.r.is_accessible_from(ctor_vis, self.parent_scope.module);
                 if !is_expected(ctor_def) || is_accessible {
@@ -1445,10 +1471,12 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                         );
 
                         // Use spans of the tuple struct definition.
-                        self.r
-                            .field_names
-                            .get(&def_id)
-                            .map(|fields| fields.iter().map(|f| f.span).collect::<Vec<_>>())
+                        self.r.field_def_ids(def_id).map(|field_ids| {
+                            field_ids
+                                .iter()
+                                .map(|&field_id| self.r.def_span(field_id))
+                                .collect::<Vec<_>>()
+                        })
                     }
                     _ => None,
                 };
@@ -1494,9 +1522,10 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 match source {
                     PathSource::Expr(_) | PathSource::TupleStruct(..) | PathSource::Pat => {
                         let span = find_span(&source, err);
-                        if let Some(span) = self.def_span(def_id) {
-                            err.span_label(span, &format!("`{}` defined here", path_str));
-                        }
+                        err.span_label(
+                            self.r.def_span(def_id),
+                            &format!("`{path_str}` defined here"),
+                        );
                         err.span_suggestion(
                             span,
                             "use this syntax instead",
@@ -1508,13 +1537,11 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 }
             }
             (Res::Def(DefKind::Ctor(_, CtorKind::Fn), ctor_def_id), _) if ns == ValueNS => {
-                let def_id = self.r.parent(ctor_def_id);
-                if let Some(span) = self.def_span(def_id) {
-                    err.span_label(span, &format!("`{}` defined here", path_str));
-                }
-                let fields = self.r.field_names.get(&def_id).map_or_else(
+                let def_id = self.r.tcx.parent(ctor_def_id);
+                err.span_label(self.r.def_span(def_id), &format!("`{path_str}` defined here"));
+                let fields = self.r.field_def_ids(def_id).map_or_else(
                     || "/* fields */".to_string(),
-                    |fields| vec!["_"; fields.len()].join(", "),
+                    |field_ids| vec!["_"; field_ids.len()].join(", "),
                 );
                 err.span_suggestion(
                     span,
@@ -1595,8 +1622,11 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                     if let Some(Res::Def(DefKind::Struct | DefKind::Union, did)) =
                         resolution.full_res()
                     {
-                        if let Some(field_names) = self.r.field_names.get(&did) {
-                            if field_names.iter().any(|&field_name| ident.name == field_name.node) {
+                        if let Some(field_ids) = self.r.field_def_ids(did) {
+                            if field_ids
+                                .iter()
+                                .any(|&field_id| ident.name == self.r.tcx.item_name(field_id))
+                            {
                                 return Some(AssocSuggestion::Field);
                             }
                         }
@@ -1631,7 +1661,17 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
             ) {
                 let res = binding.res();
                 if filter_fn(res) {
-                    if self.r.has_self.contains(&res.def_id()) {
+                    let def_id = res.def_id();
+                    let has_self = match def_id.as_local() {
+                        Some(def_id) => self.r.has_self.contains(&def_id),
+                        None => self
+                            .r
+                            .tcx
+                            .fn_arg_names(def_id)
+                            .first()
+                            .map_or(false, |ident| ident.name == kw::SelfLower),
+                    };
+                    if has_self {
                         return Some(AssocSuggestion::MethodWithSelf { called });
                     } else {
                         match res {
@@ -1737,7 +1777,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
 
         let name = path[path.len() - 1].ident.name;
         // Make sure error reporting is deterministic.
-        names.sort_by(|a, b| a.candidate.as_str().partial_cmp(b.candidate.as_str()).unwrap());
+        names.sort_by(|a, b| a.candidate.as_str().cmp(b.candidate.as_str()));
 
         match find_best_match_for_name(
             &names.iter().map(|suggestion| suggestion.candidate).collect::<Vec<Symbol>>(),
@@ -1999,12 +2039,13 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
             }
         } else {
             let needs_placeholder = |ctor_def_id: DefId, kind: CtorKind| {
-                let def_id = self.r.parent(ctor_def_id);
-                let has_no_fields = self.r.field_names.get(&def_id).map_or(false, |f| f.is_empty());
+                let def_id = self.r.tcx.parent(ctor_def_id);
                 match kind {
                     CtorKind::Const => false,
-                    CtorKind::Fn if has_no_fields => false,
-                    _ => true,
+                    CtorKind::Fn => !self
+                        .r
+                        .field_def_ids(def_id)
+                        .map_or(false, |field_ids| field_ids.is_empty()),
                 }
             };
 
@@ -2065,15 +2106,14 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
         };
 
         if def_id.is_local() {
-            if let Some(span) = self.def_span(def_id) {
-                err.span_note(span, "the enum is defined here");
-            }
+            err.span_note(self.r.def_span(def_id), "the enum is defined here");
         }
     }
 
-    pub(crate) fn report_missing_type_error(
+    pub(crate) fn suggest_adding_generic_parameter(
         &self,
         path: &[Segment],
+        source: PathSource<'_>,
     ) -> Option<(Span, &'static str, String, Applicability)> {
         let (ident, span) = match path {
             [segment]
@@ -2109,7 +2149,6 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
             // Without the 2nd `true`, we'd suggest `impl <T>` for `impl T` when a type `T` isn't found
             | (Some(Item { kind: kind @ ItemKind::Impl(..), .. }), true, true)
             | (Some(Item { kind, .. }), false, _) => {
-                // Likely missing type parameter.
                 if let Some(generics) = kind.generics() {
                     if span.overlaps(generics.span) {
                         // Avoid the following:
@@ -2122,7 +2161,12 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                         //   |           not found in this scope
                         return None;
                     }
-                    let msg = "you might be missing a type parameter";
+
+                    let (msg, sugg) = match source {
+                        PathSource::Type => ("you might be missing a type parameter", ident),
+                        PathSource::Expr(_) => ("you might be missing a const parameter", format!("const {ident}: /* Type */")),
+                        _ => return None,
+                    };
                     let (span, sugg) = if let [.., param] = &generics.params[..] {
                         let span = if let [.., bound] = &param.bounds[..] {
                             bound.span()
@@ -2133,9 +2177,9 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                         } else {
                             param.ident.span
                         };
-                        (span, format!(", {}", ident))
+                        (span, format!(", {sugg}"))
                     } else {
-                        (generics.span, format!("<{}>", ident))
+                        (generics.span, format!("<{sugg}>"))
                     };
                     // Do not suggest if this is coming from macro expansion.
                     if span.can_be_used_for_suggestions() {
diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs
index 1fdfb1a53..b820d56b8 100644
--- a/compiler/rustc_resolve/src/lib.rs
+++ b/compiler/rustc_resolve/src/lib.rs
@@ -23,10 +23,11 @@ extern crate tracing;
 
 use rustc_arena::{DroplessArena, TypedArena};
 use rustc_ast::node_id::NodeMap;
-use rustc_ast::{self as ast, NodeId, CRATE_NODE_ID};
+use rustc_ast::{self as ast, attr, NodeId, CRATE_NODE_ID};
 use rustc_ast::{AngleBracketedArg, Crate, Expr, ExprKind, GenericArg, GenericArgs, LitKind, Path};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::intern::Interned;
+use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{Lrc, MappedReadGuard};
 use rustc_errors::{
     Applicability, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed, SubdiagnosticMessage,
@@ -34,7 +35,7 @@ use rustc_errors::{
 use rustc_expand::base::{DeriveResolutions, SyntaxExtension, SyntaxExtensionKind};
 use rustc_hir::def::Namespace::{self, *};
 use rustc_hir::def::{self, CtorOf, DefKind, DocLinkResMap, LifetimeRes, PartialRes, PerNS};
-use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId};
+use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LocalDefIdMap, LocalDefIdSet};
 use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::definitions::DefPathData;
 use rustc_hir::TraitCandidate;
@@ -44,13 +45,11 @@ use rustc_metadata::creader::{CStore, CrateLoader};
 use rustc_middle::metadata::ModChild;
 use rustc_middle::middle::privacy::EffectiveVisibilities;
 use rustc_middle::span_bug;
-use rustc_middle::ty::{self, DefIdTree, MainDefinition, RegisteredTools, TyCtxt};
+use rustc_middle::ty::{self, MainDefinition, RegisteredTools, TyCtxt};
 use rustc_middle::ty::{ResolverGlobalCtxt, ResolverOutputs};
 use rustc_query_system::ich::StableHashingContext;
-use rustc_session::cstore::CrateStore;
 use rustc_session::lint::LintBuffer;
 use rustc_span::hygiene::{ExpnId, LocalExpnId, MacroKind, SyntaxContext, Transparency};
-use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 
@@ -80,7 +79,7 @@ mod late;
 mod macros;
 pub mod rustdoc;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 enum Weak {
     Yes,
@@ -880,11 +879,8 @@ pub struct Resolver<'a, 'tcx> {
     extern_prelude: FxHashMap<Ident, ExternPreludeEntry<'a>>,
 
     /// N.B., this is used only for better diagnostics, not name resolution itself.
-    has_self: FxHashSet<DefId>,
-
-    /// Names of fields of an item `DefId` accessible with dot syntax.
-    /// Used for hints during error reporting.
-    field_names: FxHashMap<DefId, Vec<Spanned<Symbol>>>,
+    has_self: LocalDefIdSet,
+    field_def_ids: LocalDefIdMap<&'tcx [DefId]>,
 
     /// Span of the privacy modifier in fields of an item `DefId` accessible with dot syntax.
     /// Used for hints during error reporting.
@@ -913,7 +909,8 @@ pub struct Resolver<'a, 'tcx> {
 
     /// `CrateNum` resolutions of `extern crate` items.
     extern_crate_map: FxHashMap<LocalDefId, CrateNum>,
-    reexport_map: FxHashMap<LocalDefId, Vec<ModChild>>,
+    module_children_non_reexports: LocalDefIdMap<Vec<LocalDefId>>,
+    module_children_reexports: LocalDefIdMap<Vec<ModChild>>,
     trait_map: NodeMap<Vec<TraitCandidate>>,
 
     /// A map from nodes to anonymous modules.
@@ -965,7 +962,7 @@ pub struct Resolver<'a, 'tcx> {
     /// A small map keeping true kinds of built-in macros that appear to be fn-like on
     /// the surface (`macro` items in libcore), but are actually attributes or derives.
     builtin_macro_kinds: FxHashMap<LocalDefId, MacroKind>,
-    registered_tools: RegisteredTools,
+    registered_tools: &'tcx RegisteredTools,
     macro_use_prelude: FxHashMap<Symbol, &'a NameBinding<'a>>,
     macro_map: FxHashMap<DefId, MacroData>,
     dummy_ext_bang: Lrc<SyntaxExtension>,
@@ -1008,7 +1005,7 @@ pub struct Resolver<'a, 'tcx> {
     /// Table for mapping struct IDs into struct constructor IDs,
     /// it's not used during normal resolution, only for better error reporting.
     /// Also includes of list of each fields visibility
-    struct_constructors: DefIdMap<(Res, ty::Visibility<DefId>, Vec<ty::Visibility<DefId>>)>,
+    struct_constructors: LocalDefIdMap<(Res, ty::Visibility<DefId>, Vec<ty::Visibility<DefId>>)>,
 
     /// Features enabled for this crate.
     active_features: FxHashSet<Symbol>,
@@ -1117,13 +1114,6 @@ impl<'a, 'tcx> AsMut<Resolver<'a, 'tcx>> for Resolver<'a, 'tcx> {
     }
 }
 
-impl<'a, 'b, 'tcx> DefIdTree for &'a Resolver<'b, 'tcx> {
-    #[inline]
-    fn opt_parent(self, id: DefId) -> Option<DefId> {
-        self.tcx.opt_parent(id)
-    }
-}
-
 impl<'tcx> Resolver<'_, 'tcx> {
     fn opt_local_def_id(&self, node: NodeId) -> Option<LocalDefId> {
         self.node_id_to_def_id.get(&node).copied()
@@ -1179,7 +1169,7 @@ impl<'tcx> Resolver<'_, 'tcx> {
         if let Some(def_id) = def_id.as_local() {
             self.item_generics_num_lifetimes[&def_id]
         } else {
-            self.cstore().item_generics_num_lifetimes(def_id, self.tcx.sess)
+            self.tcx.generics_of(def_id).own_counts().lifetimes
         }
     }
 
@@ -1191,7 +1181,8 @@ impl<'tcx> Resolver<'_, 'tcx> {
 impl<'a, 'tcx> Resolver<'a, 'tcx> {
     pub fn new(
         tcx: TyCtxt<'tcx>,
-        krate: &Crate,
+        attrs: &[ast::Attribute],
+        crate_span: Span,
         arenas: &'a ResolverArenas<'a>,
     ) -> Resolver<'a, 'tcx> {
         let root_def_id = CRATE_DEF_ID.to_def_id();
@@ -1200,8 +1191,8 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             None,
             ModuleKind::Def(DefKind::Mod, root_def_id, kw::Empty),
             ExpnId::root(),
-            krate.spans.inner_span,
-            tcx.sess.contains_name(&krate.attrs, sym::no_implicit_prelude),
+            crate_span,
+            attr::contains_name(attrs, sym::no_implicit_prelude),
             &mut module_map,
         );
         let empty_module = arenas.new_module(
@@ -1233,14 +1224,14 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             .map(|(name, _)| (Ident::from_str(name), Default::default()))
             .collect();
 
-        if !tcx.sess.contains_name(&krate.attrs, sym::no_core) {
+        if !attr::contains_name(attrs, sym::no_core) {
             extern_prelude.insert(Ident::with_dummy_span(sym::core), Default::default());
-            if !tcx.sess.contains_name(&krate.attrs, sym::no_std) {
+            if !attr::contains_name(attrs, sym::no_std) {
                 extern_prelude.insert(Ident::with_dummy_span(sym::std), Default::default());
             }
         }
 
-        let registered_tools = macros::registered_tools(tcx.sess, &krate.attrs);
+        let registered_tools = tcx.registered_tools(());
 
         let features = tcx.sess.features_untracked();
 
@@ -1255,8 +1246,8 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             prelude: None,
             extern_prelude,
 
-            has_self: FxHashSet::default(),
-            field_names: FxHashMap::default(),
+            has_self: Default::default(),
+            field_def_ids: Default::default(),
             field_visibility_spans: FxHashMap::default(),
 
             determined_imports: Vec::new(),
@@ -1269,7 +1260,8 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             lifetimes_res_map: Default::default(),
             extra_lifetime_params_map: Default::default(),
             extern_crate_map: Default::default(),
-            reexport_map: FxHashMap::default(),
+            module_children_non_reexports: Default::default(),
+            module_children_reexports: Default::default(),
             trait_map: NodeMap::default(),
             underscore_disambiguator: 0,
             empty_module,
@@ -1396,7 +1388,6 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         let visibilities = self.visibilities;
         let has_pub_restricted = self.has_pub_restricted;
         let extern_crate_map = self.extern_crate_map;
-        let reexport_map = self.reexport_map;
         let maybe_unused_trait_imports = self.maybe_unused_trait_imports;
         let glob_map = self.glob_map;
         let main_def = self.main_def;
@@ -1408,14 +1399,14 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             has_pub_restricted,
             effective_visibilities,
             extern_crate_map,
-            reexport_map,
+            module_children_non_reexports: self.module_children_non_reexports,
+            module_children_reexports: self.module_children_reexports,
             glob_map,
             maybe_unused_trait_imports,
             main_def,
             trait_impls: self.trait_impls,
             proc_macros,
             confused_type_with_std_module,
-            registered_tools: self.registered_tools,
             doc_link_resolutions: self.doc_link_resolutions,
             doc_link_traits_in_scope: self.doc_link_traits_in_scope,
             all_macro_rules: self.all_macro_rules,
@@ -1433,6 +1424,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             trait_map: self.trait_map,
             builtin_macro_kinds: self.builtin_macro_kinds,
             lifetime_elision_allowed: self.lifetime_elision_allowed,
+            lint_buffer: Steal::new(self.lint_buffer),
         };
         ResolverOutputs { global_ctxt, ast_lowering }
     }
@@ -1442,9 +1434,11 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
     }
 
     fn crate_loader<T>(&mut self, f: impl FnOnce(&mut CrateLoader<'_, '_>) -> T) -> T {
-        let mut cstore = self.tcx.untracked().cstore.write();
-        let cstore = cstore.untracked_as_any().downcast_mut().unwrap();
-        f(&mut CrateLoader::new(self.tcx, &mut *cstore, &mut self.used_extern_options))
+        f(&mut CrateLoader::new(
+            self.tcx,
+            &mut CStore::from_tcx_mut(self.tcx),
+            &mut self.used_extern_options,
+        ))
     }
 
     fn cstore(&self) -> MappedReadGuard<'_, CStore> {
@@ -1483,9 +1477,12 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
     pub fn resolve_crate(&mut self, krate: &Crate) {
         self.tcx.sess.time("resolve_crate", || {
             self.tcx.sess.time("finalize_imports", || self.finalize_imports());
-            self.tcx.sess.time("compute_effective_visibilities", || {
+            let exported_ambiguities = self.tcx.sess.time("compute_effective_visibilities", || {
                 EffectiveVisibilitiesVisitor::compute_effective_visibilities(self, krate)
             });
+            self.tcx.sess.time("check_reexport_ambiguities", || {
+                self.check_reexport_ambiguities(exported_ambiguities)
+            });
             self.tcx.sess.time("finalize_macro_resolutions", || self.finalize_macro_resolutions());
             self.tcx.sess.time("late_resolve_crate", || self.late_resolve_crate(krate));
             self.tcx.sess.time("resolve_main", || self.resolve_main());
@@ -1656,7 +1653,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                 misc2: AmbiguityErrorMisc::None,
             };
             if !self.matches_previous_ambiguity_error(&ambiguity_error) {
-                // avoid dumplicated span information to be emitt out
+                // avoid duplicated span information to be emitt out
                 self.ambiguity_errors.push(ambiguity_error);
             }
         }
@@ -1789,7 +1786,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         vis: ty::Visibility<impl Into<DefId>>,
         module: Module<'a>,
     ) -> bool {
-        vis.is_accessible_from(module.nearest_parent_mod(), self)
+        vis.is_accessible_from(module.nearest_parent_mod(), self.tcx)
     }
 
     fn set_binding_parent_module(&mut self, binding: &'a NameBinding<'a>, module: Module<'a>) {
@@ -1856,20 +1853,12 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         &mut self,
         path_str: &str,
         ns: Namespace,
-        mut parent_scope: ParentScope<'a>,
+        parent_scope: ParentScope<'a>,
     ) -> Option<Res> {
         let mut segments =
             Vec::from_iter(path_str.split("::").map(Ident::from_str).map(Segment::from_ident));
         if let Some(segment) = segments.first_mut() {
-            if segment.ident.name == kw::Crate {
-                // FIXME: `resolve_path` always resolves `crate` to the current crate root, but
-                // rustdoc wants it to resolve to the `parent_scope`'s crate root. This trick of
-                // replacing `crate` with `self` and changing the current module should achieve
-                // the same effect.
-                segment.ident.name = kw::SelfLower;
-                parent_scope.module =
-                    self.expect_module(parent_scope.module.def_id().krate.as_def_id());
-            } else if segment.ident.name == kw::Empty {
+            if segment.ident.name == kw::Empty {
                 segment.ident.name = kw::PathRoot;
             }
         }
@@ -1884,20 +1873,20 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         }
     }
 
-    /// Retrieves the span of the given `DefId` if `DefId` is in the local crate.
-    #[inline]
-    fn opt_span(&self, def_id: DefId) -> Option<Span> {
-        def_id.as_local().map(|def_id| self.tcx.source_span(def_id))
+    /// Retrieves definition span of the given `DefId`.
+    fn def_span(&self, def_id: DefId) -> Span {
+        match def_id.as_local() {
+            Some(def_id) => self.tcx.source_span(def_id),
+            // Query `def_span` is not used because hashing its result span is expensive.
+            None => self.cstore().def_span_untracked(def_id, self.tcx.sess),
+        }
     }
 
-    /// Retrieves the name of the given `DefId`.
-    #[inline]
-    fn opt_name(&self, def_id: DefId) -> Option<Symbol> {
-        let def_key = match def_id.as_local() {
-            Some(def_id) => self.tcx.definitions_untracked().def_key(def_id),
-            None => self.cstore().def_key(def_id),
-        };
-        def_key.get_opt_name()
+    fn field_def_ids(&self, def_id: DefId) -> Option<&'tcx [DefId]> {
+        match def_id.as_local() {
+            Some(def_id) => self.field_def_ids.get(&def_id).copied(),
+            None => Some(self.tcx.associated_item_def_ids(def_id)),
+        }
     }
 
     /// Checks if an expression refers to a function marked with
@@ -1924,10 +1913,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                     return v.clone();
                 }
 
-                let attr = self
-                    .cstore()
-                    .item_attrs_untracked(def_id, self.tcx.sess)
-                    .find(|a| a.has_name(sym::rustc_legacy_const_generics))?;
+                let attr = self.tcx.get_attr(def_id, sym::rustc_legacy_const_generics)?;
                 let mut ret = Vec::new();
                 for meta in attr.meta_item_list()? {
                     match meta.lit()?.kind {
@@ -1965,20 +1951,6 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         }
         self.main_def = Some(MainDefinition { res, is_import, span });
     }
-
-    // Items that go to reexport table encoded to metadata and visible through it to other crates.
-    fn is_reexport(&self, binding: &NameBinding<'a>) -> Option<def::Res<!>> {
-        if binding.is_import() {
-            let res = binding.res().expect_non_local();
-            // Ambiguous imports are treated as errors at this point and are
-            // not exposed to other crates (see #36837 for more details).
-            if res != def::Res::Err && !binding.is_ambiguity() {
-                return Some(res);
-            }
-        }
-
-        return None;
-    }
 }
 
 fn names_to_string(names: &[Symbol]) -> String {
@@ -2047,3 +2019,7 @@ impl Finalize {
         Finalize { node_id, path_span, root_span, report_private: true }
     }
 }
+
+pub fn provide(providers: &mut ty::query::Providers) {
+    providers.registered_tools = macros::registered_tools;
+}
diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs
index b38c11e8b..22b014c06 100644
--- a/compiler/rustc_resolve/src/macros.rs
+++ b/compiler/rustc_resolve/src/macros.rs
@@ -1,14 +1,14 @@
 //! A bunch of methods and structures more or less related to resolving macros and
 //! interface provided by `Resolver` to macro expander.
 
+use crate::errors::{AddAsNonDerive, MacroExpectedFound, RemoveSurroundingDerive};
 use crate::Namespace::*;
 use crate::{BuiltinMacroState, Determinacy};
 use crate::{DeriveData, Finalize, ParentScope, ResolutionError, Resolver, ScopeSet};
 use crate::{ModuleKind, ModuleOrUniformRoot, NameBinding, PathResult, Segment};
-use rustc_ast::{self as ast, Inline, ItemKind, ModKind, NodeId};
+use rustc_ast::{self as ast, attr, Inline, ItemKind, ModKind, NodeId};
 use rustc_ast_pretty::pprust;
 use rustc_attr::StabilityLevel;
-use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::intern::Interned;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{struct_span_err, Applicability};
@@ -20,11 +20,11 @@ use rustc_hir::def::{self, DefKind, NonMacroAttrKind};
 use rustc_hir::def_id::{CrateNum, LocalDefId};
 use rustc_middle::middle::stability;
 use rustc_middle::ty::RegisteredTools;
+use rustc_middle::ty::TyCtxt;
 use rustc_session::lint::builtin::{LEGACY_DERIVE_HELPERS, SOFT_UNSTABLE};
 use rustc_session::lint::builtin::{UNUSED_MACROS, UNUSED_MACRO_RULES};
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_session::parse::feature_err;
-use rustc_session::Session;
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::{self, ExpnData, ExpnKind, LocalExpnId};
 use rustc_span::hygiene::{AstPass, MacroKind};
@@ -111,15 +111,17 @@ fn fast_print_path(path: &ast::Path) -> Symbol {
     }
 }
 
-pub(crate) fn registered_tools(sess: &Session, attrs: &[ast::Attribute]) -> FxHashSet<Ident> {
-    let mut registered_tools = FxHashSet::default();
-    for attr in sess.filter_by_name(attrs, sym::register_tool) {
+pub(crate) fn registered_tools(tcx: TyCtxt<'_>, (): ()) -> RegisteredTools {
+    let mut registered_tools = RegisteredTools::default();
+    let (_, pre_configured_attrs) = &*tcx.crate_for_resolver(()).borrow();
+    for attr in attr::filter_by_name(pre_configured_attrs, sym::register_tool) {
         for nested_meta in attr.meta_item_list().unwrap_or_default() {
             match nested_meta.ident() {
                 Some(ident) => {
                     if let Some(old_ident) = registered_tools.replace(ident) {
                         let msg = format!("{} `{}` was already registered", "tool", ident);
-                        sess.struct_span_err(ident.span, &msg)
+                        tcx.sess
+                            .struct_span_err(ident.span, &msg)
                             .span_label(old_ident.span, "already registered here")
                             .emit();
                     }
@@ -127,7 +129,10 @@ pub(crate) fn registered_tools(sess: &Session, attrs: &[ast::Attribute]) -> FxHa
                 None => {
                     let msg = format!("`{}` only accepts identifiers", sym::register_tool);
                     let span = nested_meta.span();
-                    sess.struct_span_err(span, &msg).span_label(span, "not an identifier").emit();
+                    tcx.sess
+                        .struct_span_err(span, &msg)
+                        .span_label(span, "not an identifier")
+                        .emit();
                 }
             }
         }
@@ -539,12 +544,29 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         };
         if let Some((article, expected)) = unexpected_res {
             let path_str = pprust::path_to_string(path);
-            let msg = format!("expected {}, found {} `{}`", expected, res.descr(), path_str);
-            self.tcx
-                .sess
-                .struct_span_err(path.span, &msg)
-                .span_label(path.span, format!("not {} {}", article, expected))
-                .emit();
+
+            let mut err = MacroExpectedFound {
+                span: path.span,
+                expected,
+                found: res.descr(),
+                macro_path: &path_str,
+                ..Default::default() // Subdiagnostics default to None
+            };
+
+            // Suggest moving the macro out of the derive() if the macro isn't Derive
+            if !path.span.from_expansion()
+                && kind == MacroKind::Derive
+                && ext.macro_kind() != MacroKind::Derive
+            {
+                err.remove_surrounding_derive = Some(RemoveSurroundingDerive { span: path.span });
+                err.add_as_non_derive = Some(AddAsNonDerive { macro_path: &path_str });
+            }
+
+            let mut err = self.tcx.sess.create_err(err);
+            err.span_label(path.span, format!("not {} {}", article, expected));
+
+            err.emit();
+
             return Ok((self.dummy_ext(kind), Res::Err));
         }
 
@@ -699,7 +721,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                 PathResult::NonModule(path_res) if let Some(res) = path_res.full_res() => {
                     check_consistency(self, &path, path_span, kind, initial_res, res)
                 }
-                path_res @ PathResult::NonModule(..) | path_res @ PathResult::Failed { .. } => {
+                path_res @ (PathResult::NonModule(..) | PathResult::Failed { .. }) => {
                     let mut suggestion = None;
                     let (span, label) = if let PathResult::Failed { span, label, .. } = path_res {
                         // try to suggest if it's not a macro, maybe a function
diff --git a/compiler/rustc_resolve/src/rustdoc.rs b/compiler/rustc_resolve/src/rustdoc.rs
index b8853c174..9eae99be2 100644
--- a/compiler/rustc_resolve/src/rustdoc.rs
+++ b/compiler/rustc_resolve/src/rustdoc.rs
@@ -3,7 +3,7 @@ use rustc_ast as ast;
 use rustc_ast::util::comments::beautify_doc_string;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_span::def_id::DefId;
-use rustc_span::symbol::{kw, Symbol};
+use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
 use std::{cmp, mem};
 
@@ -26,11 +26,13 @@ pub enum DocFragmentKind {
 #[derive(Clone, PartialEq, Eq, Debug)]
 pub struct DocFragment {
     pub span: Span,
-    /// The module this doc-comment came from.
-    ///
-    /// This allows distinguishing between the original documentation and a pub re-export.
-    /// If it is `None`, the item was not re-exported.
-    pub parent_module: Option<DefId>,
+    /// The item this doc-comment came from.
+    /// Used to determine the scope in which doc links in this fragment are resolved.
+    /// Typically filled for reexport docs when they are merged into the docs of the
+    /// original reexported item.
+    /// If the id is not filled, which happens for the original reexported item, then
+    /// it has to be taken from somewhere else during doc link resolution.
+    pub item_id: Option<DefId>,
     pub doc: Symbol,
     pub kind: DocFragmentKind,
     pub indent: usize,
@@ -186,7 +188,7 @@ pub fn attrs_to_doc_fragments<'a>(
 ) -> (Vec<DocFragment>, ast::AttrVec) {
     let mut doc_fragments = Vec::new();
     let mut other_attrs = ast::AttrVec::new();
-    for (attr, parent_module) in attrs {
+    for (attr, item_id) in attrs {
         if let Some((doc_str, comment_kind)) = attr.doc_str_and_comment_kind() {
             let doc = beautify_doc_string(doc_str, comment_kind);
             let kind = if attr.is_doc_comment() {
@@ -194,7 +196,7 @@ pub fn attrs_to_doc_fragments<'a>(
             } else {
                 DocFragmentKind::RawDoc
             };
-            let fragment = DocFragment { span: attr.span, doc, kind, parent_module, indent: 0 };
+            let fragment = DocFragment { span: attr.span, doc, kind, item_id, indent: 0 };
             doc_fragments.push(fragment);
         } else if !doc_only {
             other_attrs.push(attr.clone());
@@ -216,7 +218,7 @@ pub fn prepare_to_doc_link_resolution(
 ) -> FxHashMap<Option<DefId>, String> {
     let mut res = FxHashMap::default();
     for fragment in doc_fragments {
-        let out_str = res.entry(fragment.parent_module).or_default();
+        let out_str = res.entry(fragment.item_id).or_default();
         add_doc_fragment(out_str, fragment);
     }
     res
@@ -337,6 +339,22 @@ pub fn inner_docs(attrs: &[ast::Attribute]) -> bool {
     attrs.iter().find(|a| a.doc_str().is_some()).map_or(true, |a| a.style == ast::AttrStyle::Inner)
 }
 
+/// Has `#[rustc_doc_primitive]` or `#[doc(keyword)]`.
+pub fn has_primitive_or_keyword_docs(attrs: &[ast::Attribute]) -> bool {
+    for attr in attrs {
+        if attr.has_name(sym::rustc_doc_primitive) {
+            return true;
+        } else if attr.has_name(sym::doc) && let Some(items) = attr.meta_item_list() {
+            for item in items {
+                if item.has_name(sym::keyword) {
+                    return true;
+                }
+            }
+        }
+    }
+    false
+}
+
 /// Simplified version of the corresponding function in rustdoc.
 /// If the rustdoc version returns a successful result, this function must return the same result.
 /// Otherwise this function may return anything.
diff --git a/compiler/rustc_serialize/Cargo.toml b/compiler/rustc_serialize/Cargo.toml
index c04465719..e4dbb8a63 100644
--- a/compiler/rustc_serialize/Cargo.toml
+++ b/compiler/rustc_serialize/Cargo.toml
@@ -4,7 +4,7 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-indexmap = "1.9.1"
+indexmap = "1.9.3"
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
 
diff --git a/compiler/rustc_serialize/src/opaque.rs b/compiler/rustc_serialize/src/opaque.rs
index 0e0ebc79e..53e5c8967 100644
--- a/compiler/rustc_serialize/src/opaque.rs
+++ b/compiler/rustc_serialize/src/opaque.rs
@@ -123,18 +123,6 @@ impl Encoder for MemEncoder {
     }
 
     #[inline]
-    fn emit_f64(&mut self, v: f64) {
-        let as_u64: u64 = v.to_bits();
-        self.emit_u64(as_u64);
-    }
-
-    #[inline]
-    fn emit_f32(&mut self, v: f32) {
-        let as_u32: u32 = v.to_bits();
-        self.emit_u32(as_u32);
-    }
-
-    #[inline]
     fn emit_char(&mut self, v: char) {
         self.emit_u32(v as u32);
     }
@@ -501,18 +489,6 @@ impl Encoder for FileEncoder {
     }
 
     #[inline]
-    fn emit_f64(&mut self, v: f64) {
-        let as_u64: u64 = v.to_bits();
-        self.emit_u64(as_u64);
-    }
-
-    #[inline]
-    fn emit_f32(&mut self, v: f32) {
-        let as_u32: u32 = v.to_bits();
-        self.emit_u32(as_u32);
-    }
-
-    #[inline]
     fn emit_char(&mut self, v: char) {
         self.emit_u32(v as u32);
     }
@@ -643,18 +619,6 @@ impl<'a> Decoder for MemDecoder<'a> {
     }
 
     #[inline]
-    fn read_f64(&mut self) -> f64 {
-        let bits = self.read_u64();
-        f64::from_bits(bits)
-    }
-
-    #[inline]
-    fn read_f32(&mut self) -> f32 {
-        let bits = self.read_u32();
-        f32::from_bits(bits)
-    }
-
-    #[inline]
     fn read_char(&mut self) -> char {
         let bits = self.read_u32();
         std::char::from_u32(bits).unwrap()
diff --git a/compiler/rustc_serialize/src/serialize.rs b/compiler/rustc_serialize/src/serialize.rs
index 567fe0610..527abc237 100644
--- a/compiler/rustc_serialize/src/serialize.rs
+++ b/compiler/rustc_serialize/src/serialize.rs
@@ -22,6 +22,11 @@ use std::sync::Arc;
 /// be processed or ignored, whichever is appropriate. Then they should provide
 /// a `finish` method that finishes up encoding. If the encoder is fallible,
 /// `finish` should return a `Result` that indicates success or failure.
+///
+/// This current does not support `f32` nor `f64`, as they're not needed in any
+/// serialized data structures. That could be changed, but consider whether it
+/// really makes sense to store floating-point values at all.
+/// (If you need it, revert <https://github.com/rust-lang/rust/pull/109984>.)
 pub trait Encoder {
     // Primitive types:
     fn emit_usize(&mut self, v: usize);
@@ -37,8 +42,6 @@ pub trait Encoder {
     fn emit_i16(&mut self, v: i16);
     fn emit_i8(&mut self, v: i8);
     fn emit_bool(&mut self, v: bool);
-    fn emit_f64(&mut self, v: f64);
-    fn emit_f32(&mut self, v: f32);
     fn emit_char(&mut self, v: char);
     fn emit_str(&mut self, v: &str);
     fn emit_raw_bytes(&mut self, s: &[u8]);
@@ -58,6 +61,11 @@ pub trait Encoder {
 // top-level invocation would also just panic on failure. Switching to
 // infallibility made things faster and lots of code a little simpler and more
 // concise.
+///
+/// This current does not support `f32` nor `f64`, as they're not needed in any
+/// serialized data structures. That could be changed, but consider whether it
+/// really makes sense to store floating-point values at all.
+/// (If you need it, revert <https://github.com/rust-lang/rust/pull/109984>.)
 pub trait Decoder {
     // Primitive types:
     fn read_usize(&mut self) -> usize;
@@ -73,8 +81,6 @@ pub trait Decoder {
     fn read_i16(&mut self) -> i16;
     fn read_i8(&mut self) -> i8;
     fn read_bool(&mut self) -> bool;
-    fn read_f64(&mut self) -> f64;
-    fn read_f32(&mut self) -> f32;
     fn read_char(&mut self) -> char;
     fn read_str(&mut self) -> &str;
     fn read_raw_bytes(&mut self, len: usize) -> &[u8];
@@ -143,8 +149,6 @@ direct_serialize_impls! {
     i64 emit_i64 read_i64,
     i128 emit_i128 read_i128,
 
-    f32 emit_f32 read_f32,
-    f64 emit_f64 read_f64,
     bool emit_bool read_bool,
     char emit_char read_char
 }
diff --git a/compiler/rustc_serialize/tests/opaque.rs b/compiler/rustc_serialize/tests/opaque.rs
index 3a695d071..5e7dd18aa 100644
--- a/compiler/rustc_serialize/tests/opaque.rs
+++ b/compiler/rustc_serialize/tests/opaque.rs
@@ -22,8 +22,6 @@ struct Struct {
 
     l: char,
     m: String,
-    n: f32,
-    o: f64,
     p: bool,
     q: Option<u32>,
 }
@@ -120,24 +118,6 @@ fn test_bool() {
 }
 
 #[test]
-fn test_f32() {
-    let mut vec = vec![];
-    for i in -100..100 {
-        vec.push((i as f32) / 3.0);
-    }
-    check_round_trip(vec);
-}
-
-#[test]
-fn test_f64() {
-    let mut vec = vec![];
-    for i in -100..100 {
-        vec.push((i as f64) / 3.0);
-    }
-    check_round_trip(vec);
-}
-
-#[test]
 fn test_char() {
     let vec = vec!['a', 'b', 'c', 'd', 'A', 'X', ' ', '#', 'Ö', 'Ä', 'µ', '€'];
     check_round_trip(vec);
@@ -200,8 +180,6 @@ fn test_struct() {
 
         l: 'x',
         m: "abc".to_string(),
-        n: 20.5,
-        o: 21.5,
         p: false,
         q: None,
     }]);
@@ -222,8 +200,6 @@ fn test_struct() {
 
         l: 'y',
         m: "def".to_string(),
-        n: -20.5,
-        o: -21.5,
         p: true,
         q: Some(1234567),
     }]);
@@ -232,7 +208,7 @@ fn test_struct() {
 #[derive(PartialEq, Clone, Debug, Encodable, Decodable)]
 enum Enum {
     Variant1,
-    Variant2(usize, f32),
+    Variant2(usize, u32),
     Variant3 { a: i32, b: char, c: bool },
 }
 
@@ -240,7 +216,7 @@ enum Enum {
 fn test_enum() {
     check_round_trip(vec![
         Enum::Variant1,
-        Enum::Variant2(1, 2.5),
+        Enum::Variant2(1, 25),
         Enum::Variant3 { a: 3, b: 'b', c: false },
         Enum::Variant3 { a: -4, b: 'f', c: true },
     ]);
@@ -269,8 +245,8 @@ fn test_hash_map() {
 
 #[test]
 fn test_tuples() {
-    check_round_trip(vec![('x', (), false, 0.5f32)]);
-    check_round_trip(vec![(9i8, 10u16, 1.5f64)]);
+    check_round_trip(vec![('x', (), false, 5u32)]);
+    check_round_trip(vec![(9i8, 10u16, 15i64)]);
     check_round_trip(vec![(-12i16, 11u8, 12usize)]);
     check_round_trip(vec![(1234567isize, 100000000000000u64, 99999999999999i64)]);
     check_round_trip(vec![(String::new(), "some string".to_string())]);
diff --git a/compiler/rustc_session/Cargo.toml b/compiler/rustc_session/Cargo.toml
index d8db86c5f..9e337dde9 100644
--- a/compiler/rustc_session/Cargo.toml
+++ b/compiler/rustc_session/Cargo.toml
@@ -24,5 +24,9 @@ termize = "0.1.1"
 [target.'cfg(unix)'.dependencies]
 libc = "0.2"
 
-[target.'cfg(windows)'.dependencies]
-winapi = { version = "0.3", features = ["libloaderapi"] }
+[target.'cfg(windows)'.dependencies.windows]
+version = "0.46.0"
+features = [
+    "Win32_Foundation",
+    "Win32_System_LibraryLoader",
+]
diff --git a/compiler/rustc_session/locales/en-US.ftl b/compiler/rustc_session/messages.ftl
index ff53f22d4..ff53f22d4 100644
--- a/compiler/rustc_session/locales/en-US.ftl
+++ b/compiler/rustc_session/messages.ftl
diff --git a/compiler/rustc_session/src/code_stats.rs b/compiler/rustc_session/src/code_stats.rs
index 551782504..0dfee92f4 100644
--- a/compiler/rustc_session/src/code_stats.rs
+++ b/compiler/rustc_session/src/code_stats.rs
@@ -2,7 +2,7 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::sync::Lock;
 use rustc_span::Symbol;
 use rustc_target::abi::{Align, Size};
-use std::cmp::{self, Ordering};
+use std::cmp;
 
 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
 pub struct VariantInfo {
@@ -87,7 +87,7 @@ impl CodeStats {
         // Except for Generators, whose variants are already sorted according to
         // their yield points in `variant_info_for_generator`.
         if kind != DataTypeKind::Generator {
-            variants.sort_by(|info1, info2| info2.size.cmp(&info1.size));
+            variants.sort_by_key(|info| cmp::Reverse(info.size));
         }
         let info = TypeSizeInfo {
             kind,
@@ -107,13 +107,7 @@ impl CodeStats {
 
         // Primary sort: large-to-small.
         // Secondary sort: description (dictionary order)
-        sorted.sort_by(|info1, info2| {
-            // (reversing cmp order to get large-to-small ordering)
-            match info2.overall_size.cmp(&info1.overall_size) {
-                Ordering::Equal => info1.type_description.cmp(&info2.type_description),
-                other => other,
-            }
-        });
+        sorted.sort_by_key(|info| (cmp::Reverse(info.overall_size), &info.type_description));
 
         for info in sorted {
             let TypeSizeInfo { type_description, overall_size, align, kind, variants, .. } = info;
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index d4e4ace88..79eb31bb1 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -222,7 +222,7 @@ impl LinkerPluginLto {
 }
 
 /// The different settings that can be enabled via the `-Z location-detail` flag.
-#[derive(Clone, PartialEq, Hash, Debug)]
+#[derive(Copy, Clone, PartialEq, Hash, Debug)]
 pub struct LocationDetail {
     pub file: bool,
     pub line: bool,
@@ -260,6 +260,8 @@ pub enum SymbolManglingVersion {
 #[derive(Clone, Copy, Debug, PartialEq, Hash)]
 pub enum DebugInfo {
     None,
+    LineDirectivesOnly,
+    LineTablesOnly,
     Limited,
     Full,
 }
@@ -580,6 +582,7 @@ pub enum PrintRequest {
     CodeModels,
     TlsModels,
     TargetSpec,
+    AllTargetSpecs,
     NativeStaticLibs,
     StackProtectorStrategies,
     LinkArgs,
@@ -1137,7 +1140,7 @@ impl CrateCheckConfig {
     }
 
     /// Fills a `CrateCheckConfig` with well-known configuration values.
-    fn fill_well_known_values(&mut self) {
+    fn fill_well_known_values(&mut self, current_target: &Target) {
         if !self.well_known_values {
             return;
         }
@@ -1229,6 +1232,7 @@ impl CrateCheckConfig {
             for target in TARGETS
                 .iter()
                 .map(|target| Target::expect_builtin(&TargetTriple::from_triple(target)))
+                .chain(iter::once(current_target.clone()))
             {
                 values_target_os.insert(Symbol::intern(&target.options.os));
                 values_target_family
@@ -1243,9 +1247,9 @@ impl CrateCheckConfig {
         }
     }
 
-    pub fn fill_well_known(&mut self) {
+    pub fn fill_well_known(&mut self, current_target: &Target) {
         self.fill_well_known_names();
-        self.fill_well_known_values();
+        self.fill_well_known_values(current_target);
     }
 }
 
@@ -1254,7 +1258,7 @@ pub fn build_configuration(sess: &Session, mut user_cfg: CrateConfig) -> CrateCo
     // some default and generated configuration items.
     let default_cfg = default_configuration(sess);
     // If the user wants a test runner, then add the test cfg.
-    if sess.opts.test {
+    if sess.is_test_crate() {
         user_cfg.insert((sym::test, None));
     }
     user_cfg.extend(default_cfg.iter().cloned());
@@ -1422,7 +1426,7 @@ pub fn rustc_short_optgroups() -> Vec<RustcOptGroup> {
         opt::opt_s(
             "",
             "edition",
-            &*EDITION_STRING,
+            &EDITION_STRING,
             EDITION_NAME_LIST,
         ),
         opt::multi_s(
@@ -1438,8 +1442,8 @@ pub fn rustc_short_optgroups() -> Vec<RustcOptGroup> {
             "Compiler information to print on stdout",
             "[crate-name|file-names|sysroot|target-libdir|cfg|calling-conventions|\
              target-list|target-cpus|target-features|relocation-models|code-models|\
-             tls-models|target-spec-json|native-static-libs|stack-protector-strategies|\
-             link-args]",
+             tls-models|target-spec-json|all-target-specs-json|native-static-libs|\
+             stack-protector-strategies|link-args]",
         ),
         opt::flagmulti_s("g", "", "Equivalent to -C debuginfo=2"),
         opt::flagmulti_s("O", "", "Equivalent to -C opt-level=2"),
@@ -1886,6 +1890,7 @@ fn collect_print_requests(
         ("native-static-libs", PrintRequest::NativeStaticLibs),
         ("stack-protector-strategies", PrintRequest::StackProtectorStrategies),
         ("target-spec-json", PrintRequest::TargetSpec),
+        ("all-target-specs-json", PrintRequest::AllTargetSpecs),
         ("link-args", PrintRequest::LinkArgs),
         ("split-debuginfo", PrintRequest::SplitDebuginfo),
     ];
@@ -1899,7 +1904,18 @@ fn collect_print_requests(
                     early_error(
                         error_format,
                         "the `-Z unstable-options` flag must also be passed to \
-                     enable the target-spec-json print option",
+                         enable the target-spec-json print option",
+                    );
+                }
+            }
+            Some((_, PrintRequest::AllTargetSpecs)) => {
+                if unstable_opts.unstable_options {
+                    PrintRequest::AllTargetSpecs
+                } else {
+                    early_error(
+                        error_format,
+                        "the `-Z unstable-options` flag must also be passed to \
+                         enable the all-target-specs-json print option",
                     );
                 }
             }
@@ -1978,11 +1994,7 @@ fn parse_opt_level(
     }
 }
 
-fn select_debuginfo(
-    matches: &getopts::Matches,
-    cg: &CodegenOptions,
-    error_format: ErrorOutputType,
-) -> DebugInfo {
+fn select_debuginfo(matches: &getopts::Matches, cg: &CodegenOptions) -> DebugInfo {
     let max_g = matches.opt_positions("g").into_iter().max();
     let max_c = matches
         .opt_strs_pos("C")
@@ -1992,24 +2004,7 @@ fn select_debuginfo(
             if let Some("debuginfo") = s.split('=').next() { Some(i) } else { None }
         })
         .max();
-    if max_g > max_c {
-        DebugInfo::Full
-    } else {
-        match cg.debuginfo {
-            0 => DebugInfo::None,
-            1 => DebugInfo::Limited,
-            2 => DebugInfo::Full,
-            arg => {
-                early_error(
-                    error_format,
-                    &format!(
-                        "debug info level needs to be between \
-                         0-2 (instead was `{arg}`)"
-                    ),
-                );
-            }
-        }
-    }
+    if max_g > max_c { DebugInfo::Full } else { cg.debuginfo }
 }
 
 pub(crate) fn parse_assert_incr_state(
@@ -2497,7 +2492,7 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
     // to use them interchangeably. See the note above (regarding `-O` and `-C opt-level`)
     // for more details.
     let debug_assertions = cg.debug_assertions.unwrap_or(opt_level == OptLevel::No);
-    let debuginfo = select_debuginfo(matches, &cg, error_format);
+    let debuginfo = select_debuginfo(matches, &cg);
 
     let mut search_paths = vec![];
     for s in &matches.opt_strs("L") {
diff --git a/compiler/rustc_session/src/cstore.rs b/compiler/rustc_session/src/cstore.rs
index 868ffdf0f..dd1721801 100644
--- a/compiler/rustc_session/src/cstore.rs
+++ b/compiler/rustc_session/src/cstore.rs
@@ -6,7 +6,7 @@ use crate::search_paths::PathKind;
 use crate::utils::NativeLibKind;
 use crate::Session;
 use rustc_ast as ast;
-use rustc_data_structures::sync::{self, AppendOnlyVec, MetadataRef, RwLock};
+use rustc_data_structures::sync::{self, AppendOnlyIndexVec, MetadataRef, RwLock};
 use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, StableCrateId, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathHash, Definitions};
 use rustc_span::hygiene::{ExpnHash, ExpnId};
@@ -67,12 +67,11 @@ pub enum LinkagePreference {
 #[derive(Debug, Encodable, Decodable, HashStable_Generic)]
 pub struct NativeLib {
     pub kind: NativeLibKind,
-    pub name: Option<Symbol>,
+    pub name: Symbol,
     /// If packed_bundled_libs enabled, actual filename of library is stored.
     pub filename: Option<Symbol>,
     pub cfg: Option<ast::MetaItem>,
     pub foreign_module: Option<DefId>,
-    pub wasm_import_module: Option<Symbol>,
     pub verbatim: Option<bool>,
     pub dll_imports: Vec<DllImport>,
 }
@@ -81,6 +80,10 @@ impl NativeLib {
     pub fn has_modifiers(&self) -> bool {
         self.verbatim.is_some() || self.kind.has_modifiers()
     }
+
+    pub fn wasm_import_module(&self) -> Option<Symbol> {
+        if self.kind == NativeLibKind::WasmImportModule { Some(self.name) } else { None }
+    }
 }
 
 /// Different ways that the PE Format can decorate a symbol name.
@@ -254,6 +257,6 @@ pub type CrateStoreDyn = dyn CrateStore + sync::Sync + sync::Send;
 pub struct Untracked {
     pub cstore: RwLock<Box<CrateStoreDyn>>,
     /// Reference span for definitions.
-    pub source_span: AppendOnlyVec<LocalDefId, Span>,
+    pub source_span: AppendOnlyIndexVec<LocalDefId, Span>,
     pub definitions: RwLock<Definitions>,
 }
diff --git a/compiler/rustc_session/src/filesearch.rs b/compiler/rustc_session/src/filesearch.rs
index f1fbf3821..7fdbd48d5 100644
--- a/compiler/rustc_session/src/filesearch.rs
+++ b/compiler/rustc_session/src/filesearch.rs
@@ -1,5 +1,6 @@
 //! A module for searching for libraries
 
+use rustc_fs_util::try_canonicalize;
 use smallvec::{smallvec, SmallVec};
 use std::env;
 use std::fs;
@@ -67,6 +68,7 @@ fn current_dll_path() -> Result<PathBuf, String> {
     use std::ffi::{CStr, OsStr};
     use std::os::unix::prelude::*;
 
+    #[cfg(not(target_os = "aix"))]
     unsafe {
         let addr = current_dll_path as usize as *mut _;
         let mut info = std::mem::zeroed();
@@ -80,6 +82,49 @@ fn current_dll_path() -> Result<PathBuf, String> {
         let os = OsStr::from_bytes(bytes);
         Ok(PathBuf::from(os))
     }
+
+    #[cfg(target_os = "aix")]
+    unsafe {
+        // On AIX, the symbol `current_dll_path` references a function descriptor.
+        // A function descriptor is consisted of (See https://reviews.llvm.org/D62532)
+        // * The address of the entry point of the function.
+        // * The TOC base address for the function.
+        // * The environment pointer.
+        // The function descriptor is in the data section.
+        let addr = current_dll_path as u64;
+        let mut buffer = vec![std::mem::zeroed::<libc::ld_info>(); 64];
+        loop {
+            if libc::loadquery(
+                libc::L_GETINFO,
+                buffer.as_mut_ptr() as *mut i8,
+                (std::mem::size_of::<libc::ld_info>() * buffer.len()) as u32,
+            ) >= 0
+            {
+                break;
+            } else {
+                if std::io::Error::last_os_error().raw_os_error().unwrap() != libc::ENOMEM {
+                    return Err("loadquery failed".into());
+                }
+                buffer.resize(buffer.len() * 2, std::mem::zeroed::<libc::ld_info>());
+            }
+        }
+        let mut current = buffer.as_mut_ptr() as *mut libc::ld_info;
+        loop {
+            let data_base = (*current).ldinfo_dataorg as u64;
+            let data_end = data_base + (*current).ldinfo_datasize;
+            if (data_base..data_end).contains(&addr) {
+                let bytes = CStr::from_ptr(&(*current).ldinfo_filename[0]).to_bytes();
+                let os = OsStr::from_bytes(bytes);
+                return Ok(PathBuf::from(os));
+            }
+            if (*current).ldinfo_next == 0 {
+                break;
+            }
+            current =
+                (current as *mut i8).offset((*current).ldinfo_next as isize) as *mut libc::ld_info;
+        }
+        return Err(format!("current dll's address {} is not in the load map", addr));
+    }
 }
 
 #[cfg(windows)]
@@ -87,42 +132,45 @@ fn current_dll_path() -> Result<PathBuf, String> {
     use std::ffi::OsString;
     use std::io;
     use std::os::windows::prelude::*;
-    use std::ptr;
 
-    use winapi::um::libloaderapi::{
-        GetModuleFileNameW, GetModuleHandleExW, GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
+    use windows::{
+        core::PCWSTR,
+        Win32::Foundation::HINSTANCE,
+        Win32::System::LibraryLoader::{
+            GetModuleFileNameW, GetModuleHandleExW, GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
+        },
     };
 
+    let mut module = HINSTANCE::default();
     unsafe {
-        let mut module = ptr::null_mut();
-        let r = GetModuleHandleExW(
+        GetModuleHandleExW(
             GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
-            current_dll_path as usize as *mut _,
+            PCWSTR(current_dll_path as *mut u16),
             &mut module,
-        );
-        if r == 0 {
-            return Err(format!("GetModuleHandleExW failed: {}", io::Error::last_os_error()));
-        }
-        let mut space = Vec::with_capacity(1024);
-        let r = GetModuleFileNameW(module, space.as_mut_ptr(), space.capacity() as u32);
-        if r == 0 {
-            return Err(format!("GetModuleFileNameW failed: {}", io::Error::last_os_error()));
-        }
-        let r = r as usize;
-        if r >= space.capacity() {
-            return Err(format!("our buffer was too small? {}", io::Error::last_os_error()));
-        }
-        space.set_len(r);
-        let os = OsString::from_wide(&space);
-        Ok(PathBuf::from(os))
+        )
+    }
+    .ok()
+    .map_err(|e| e.to_string())?;
+
+    let mut filename = vec![0; 1024];
+    let n = unsafe { GetModuleFileNameW(module, &mut filename) } as usize;
+    if n == 0 {
+        return Err(format!("GetModuleFileNameW failed: {}", io::Error::last_os_error()));
     }
+    if n >= filename.capacity() {
+        return Err(format!("our buffer was too small? {}", io::Error::last_os_error()));
+    }
+
+    filename.truncate(n);
+
+    Ok(OsString::from_wide(&filename).into())
 }
 
 pub fn sysroot_candidates() -> SmallVec<[PathBuf; 2]> {
     let target = crate::config::host_triple();
     let mut sysroot_candidates: SmallVec<[PathBuf; 2]> =
         smallvec![get_or_default_sysroot().expect("Failed finding sysroot")];
-    let path = current_dll_path().and_then(|s| s.canonicalize().map_err(|e| e.to_string()));
+    let path = current_dll_path().and_then(|s| try_canonicalize(s).map_err(|e| e.to_string()));
     if let Ok(dll) = path {
         // use `parent` twice to chop off the file name and then also the
         // directory containing the dll which should be either `lib` or `bin`.
@@ -157,7 +205,7 @@ pub fn sysroot_candidates() -> SmallVec<[PathBuf; 2]> {
 pub fn get_or_default_sysroot() -> Result<PathBuf, String> {
     // Follow symlinks. If the resolved path is relative, make it absolute.
     fn canonicalize(path: PathBuf) -> PathBuf {
-        let path = fs::canonicalize(&path).unwrap_or(path);
+        let path = try_canonicalize(&path).unwrap_or(path);
         // See comments on this target function, but the gist is that
         // gcc chokes on verbatim paths which fs::canonicalize generates
         // so we try to avoid those kinds of paths.
@@ -179,28 +227,29 @@ pub fn get_or_default_sysroot() -> Result<PathBuf, String> {
         ))?;
 
         // if `dir` points target's dir, move up to the sysroot
-        if dir.ends_with(crate::config::host_triple()) {
+        let mut sysroot_dir = if dir.ends_with(crate::config::host_triple()) {
             dir.parent() // chop off `$target`
                 .and_then(|p| p.parent()) // chop off `rustlib`
-                .and_then(|p| {
-                    // chop off `lib` (this could be also $arch dir if the host sysroot uses a
-                    // multi-arch layout like Debian or Ubuntu)
-                    match p.parent() {
-                        Some(p) => match p.file_name() {
-                            Some(f) if f == "lib" => p.parent(), // first chop went for $arch, so chop again for `lib`
-                            _ => Some(p),
-                        },
-                        None => None,
-                    }
-                })
+                .and_then(|p| p.parent()) // chop off `lib`
                 .map(|s| s.to_owned())
-                .ok_or(format!(
-                    "Could not move 3 levels upper using `parent()` on {}",
-                    dir.display()
-                ))
+                .ok_or_else(|| {
+                    format!("Could not move 3 levels upper using `parent()` on {}", dir.display())
+                })?
         } else {
-            Ok(dir.to_owned())
+            dir.to_owned()
+        };
+
+        // On multiarch linux systems, there will be multiarch directory named
+        // with the architecture(e.g `x86_64-linux-gnu`) under the `lib` directory.
+        // Which cause us to mistakenly end up in the lib directory instead of the sysroot directory.
+        if sysroot_dir.ends_with("lib") {
+            sysroot_dir =
+                sysroot_dir.parent().map(|real_sysroot| real_sysroot.to_owned()).ok_or_else(
+                    || format!("Could not move to parent path of {}", sysroot_dir.display()),
+                )?
         }
+
+        Ok(sysroot_dir)
     }
 
     // Use env::args().next() to get the path of the executable without
diff --git a/compiler/rustc_session/src/lib.rs b/compiler/rustc_session/src/lib.rs
index e1f1a5f6d..968728905 100644
--- a/compiler/rustc_session/src/lib.rs
+++ b/compiler/rustc_session/src/lib.rs
@@ -2,7 +2,7 @@
 #![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(never_type)]
-#![feature(once_cell)]
+#![feature(lazy_cell)]
 #![feature(option_get_or_insert_default)]
 #![feature(rustc_attrs)]
 #![feature(map_many_mut)]
@@ -42,7 +42,7 @@ pub mod output;
 
 pub use getopts;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 /// Requirements for a `StableHashingContext` to be used in this crate.
 /// This is a hack to allow using the `HashStable_Generic` derive macro
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index b466a3fcd..631dd0a21 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -4,6 +4,7 @@ use crate::early_error;
 use crate::lint;
 use crate::search_paths::SearchPath;
 use crate::utils::NativeLib;
+use rustc_data_structures::profiling::TimePassesFormat;
 use rustc_errors::{LanguageIdentifier, TerminalUrl};
 use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, SanitizerSet};
 use rustc_target::spec::{
@@ -365,6 +366,7 @@ mod desc {
     pub const parse_number: &str = "a number";
     pub const parse_opt_number: &str = parse_number;
     pub const parse_threads: &str = parse_number;
+    pub const parse_time_passes_format: &str = "`text` (default) or `json`";
     pub const parse_passes: &str = "a space-separated list of passes, or `all`";
     pub const parse_panic_strategy: &str = "either `unwind` or `abort`";
     pub const parse_opt_panic_strategy: &str = parse_panic_strategy;
@@ -375,6 +377,7 @@ mod desc {
     pub const parse_cfguard: &str =
         "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`";
     pub const parse_cfprotection: &str = "`none`|`no`|`n` (default), `branch`, `return`, or `full`|`yes`|`y` (equivalent to `branch` and `return`)";
+    pub const parse_debuginfo: &str = "either an integer (0, 1, 2), `none`, `line-directives-only`, `line-tables-only`, `limited`, or `full`";
     pub const parse_strip: &str = "either `none`, `debuginfo`, or `symbols`";
     pub const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavorCli::one_of();
     pub const parse_optimization_fuel: &str = "crate=integer";
@@ -765,6 +768,18 @@ mod parse {
         true
     }
 
+    pub(crate) fn parse_debuginfo(slot: &mut DebugInfo, v: Option<&str>) -> bool {
+        match v {
+            Some("0") | Some("none") => *slot = DebugInfo::None,
+            Some("line-directives-only") => *slot = DebugInfo::LineDirectivesOnly,
+            Some("line-tables-only") => *slot = DebugInfo::LineTablesOnly,
+            Some("1") | Some("limited") => *slot = DebugInfo::Limited,
+            Some("2") | Some("full") => *slot = DebugInfo::Full,
+            _ => return false,
+        }
+        true
+    }
+
     pub(crate) fn parse_linker_flavor(slot: &mut Option<LinkerFlavorCli>, v: Option<&str>) -> bool {
         match v.and_then(LinkerFlavorCli::from_str) {
             Some(lf) => *slot = Some(lf),
@@ -829,6 +844,21 @@ mod parse {
         true
     }
 
+    pub(crate) fn parse_time_passes_format(slot: &mut TimePassesFormat, v: Option<&str>) -> bool {
+        match v {
+            None => true,
+            Some("json") => {
+                *slot = TimePassesFormat::Json;
+                true
+            }
+            Some("text") => {
+                *slot = TimePassesFormat::Text;
+                true
+            }
+            Some(_) => false,
+        }
+    }
+
     pub(crate) fn parse_dump_mono_stats(slot: &mut DumpMonoStatsFormat, v: Option<&str>) -> bool {
         match v {
             None => true,
@@ -894,7 +924,7 @@ mod parse {
         let mut seen_instruction_threshold = false;
         let mut seen_skip_entry = false;
         let mut seen_skip_exit = false;
-        for option in v.into_iter().map(|v| v.split(',')).flatten() {
+        for option in v.into_iter().flat_map(|v| v.split(',')) {
             match option {
                 "always" if !seen_always && !seen_never => {
                     options.always = true;
@@ -1200,9 +1230,9 @@ options! {
         "use Windows Control Flow Guard (default: no)"),
     debug_assertions: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "explicitly enable the `cfg(debug_assertions)` directive"),
-    debuginfo: usize = (0, parse_number, [TRACKED],
-        "debug info emission level (0 = no debug info, 1 = line tables only, \
-        2 = full debug info with variable and type information; default: 0)"),
+    debuginfo: DebugInfo = (DebugInfo::None, parse_debuginfo, [TRACKED],
+        "debug info emission level (0-2, none, line-directives-only, \
+        line-tables-only, limited, or full; default: 0)"),
     default_linker_libraries: bool = (false, parse_bool, [UNTRACKED],
         "allow the linker to link its default libraries (default: no)"),
     embed_bitcode: bool = (true, parse_bool, [TRACKED],
@@ -1422,6 +1452,9 @@ options! {
     fewer_names: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "reduce memory use by retaining fewer names within compilation artifacts (LLVM-IR) \
         (default: no)"),
+    flatten_format_args: bool = (false, parse_bool, [TRACKED],
+        "flatten nested format_args!() and literals into a simplified format_args!() call \
+        (default: no)"),
     force_unstable_if_unmarked: bool = (false, parse_bool, [TRACKED],
         "force all crates to be `rustc_private` unstable (default: no)"),
     fuel: Option<(String, u64)> = (None, parse_optimization_fuel, [TRACKED],
@@ -1706,6 +1739,8 @@ options! {
         "measure time of each LLVM pass (default: no)"),
     time_passes: bool = (false, parse_bool, [UNTRACKED],
         "measure time of each rustc pass (default: no)"),
+    time_passes_format: TimePassesFormat = (TimePassesFormat::Text, parse_time_passes_format, [UNTRACKED],
+        "the format to use for -Z time-passes (`text` (default) or `json`)"),
     tiny_const_eval_limit: bool = (false, parse_bool, [TRACKED],
         "sets a tiny, non-configurable limit for const eval; useful for compiler tests"),
     #[rustc_lint_opt_deny_field_access("use `Session::tls_model` instead of this field")]
diff --git a/compiler/rustc_session/src/output.rs b/compiler/rustc_session/src/output.rs
index c3f0c4b58..fdb9fae44 100644
--- a/compiler/rustc_session/src/output.rs
+++ b/compiler/rustc_session/src/output.rs
@@ -5,7 +5,7 @@ use crate::errors::{
     InvalidCharacterInCrateName,
 };
 use crate::Session;
-use rustc_ast as ast;
+use rustc_ast::{self as ast, attr};
 use rustc_span::symbol::sym;
 use rustc_span::{Span, Symbol};
 use std::path::{Path, PathBuf};
@@ -56,7 +56,7 @@ pub fn find_crate_name(sess: &Session, attrs: &[ast::Attribute]) -> Symbol {
     // the command line over one found in the #[crate_name] attribute. If we
     // find both we ensure that they're the same later on as well.
     let attr_crate_name =
-        sess.find_by_name(attrs, sym::crate_name).and_then(|at| at.value_str().map(|s| (at, s)));
+        attr::find_by_name(attrs, sym::crate_name).and_then(|at| at.value_str().map(|s| (at, s)));
 
     if let Some(ref s) = sess.opts.crate_name {
         let s = Symbol::intern(s);
diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index 4e8c3f73e..15e27952c 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -8,7 +8,7 @@ use crate::lint::{
 };
 use rustc_ast::node_id::NodeId;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
-use rustc_data_structures::sync::{Lock, Lrc};
+use rustc_data_structures::sync::{AppendOnlyVec, AtomicBool, Lock, Lrc};
 use rustc_errors::{emitter::SilentEmitter, ColorConfig, Handler};
 use rustc_errors::{
     fallback_fluent_bundle, Diagnostic, DiagnosticBuilder, DiagnosticId, DiagnosticMessage,
@@ -84,12 +84,12 @@ impl SymbolGallery {
 
 /// Construct a diagnostic for a language feature error due to the given `span`.
 /// The `feature`'s `Symbol` is the one you used in `active.rs` and `rustc_span::symbols`.
-pub fn feature_err<'a>(
-    sess: &'a ParseSess,
+pub fn feature_err(
+    sess: &ParseSess,
     feature: Symbol,
     span: impl Into<MultiSpan>,
     explain: impl Into<DiagnosticMessage>,
-) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
     feature_err_issue(sess, feature, span, GateIssue::Language, explain)
 }
 
@@ -98,20 +98,21 @@ pub fn feature_err<'a>(
 /// This variant allows you to control whether it is a library or language feature.
 /// Almost always, you want to use this for a language feature. If so, prefer `feature_err`.
 #[track_caller]
-pub fn feature_err_issue<'a>(
-    sess: &'a ParseSess,
+pub fn feature_err_issue(
+    sess: &ParseSess,
     feature: Symbol,
     span: impl Into<MultiSpan>,
     issue: GateIssue,
     explain: impl Into<DiagnosticMessage>,
-) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
     let span = span.into();
 
     // Cancel an earlier warning for this same error, if it exists.
     if let Some(span) = span.primary_span() {
-        sess.span_diagnostic
-            .steal_diagnostic(span, StashKey::EarlySyntaxWarning)
-            .map(|err| err.cancel());
+        if let Some(err) = sess.span_diagnostic.steal_diagnostic(span, StashKey::EarlySyntaxWarning)
+        {
+            err.cancel()
+        }
     }
 
     let mut err = sess.create_err(FeatureGateError { span, explain: explain.into() });
@@ -194,7 +195,7 @@ pub struct ParseSess {
     pub edition: Edition,
     /// Places where raw identifiers were used. This is used to avoid complaining about idents
     /// clashing with keywords in new editions.
-    pub raw_identifier_spans: Lock<Vec<Span>>,
+    pub raw_identifier_spans: AppendOnlyVec<Span>,
     /// Places where identifiers that contain invalid Unicode codepoints but that look like they
     /// should be. Useful to avoid bad tokenization when encountering emoji. We group them to
     /// provide a single error per unique incorrect identifier.
@@ -208,7 +209,7 @@ pub struct ParseSess {
     pub gated_spans: GatedSpans,
     pub symbol_gallery: SymbolGallery,
     /// The parser has reached `Eof` due to an unclosed brace. Used to silence unnecessary errors.
-    pub reached_eof: Lock<bool>,
+    pub reached_eof: AtomicBool,
     /// Environment variables accessed during the build and their values when they exist.
     pub env_depinfo: Lock<FxHashSet<(Symbol, Option<Symbol>)>>,
     /// File paths accessed during the build.
@@ -219,7 +220,7 @@ pub struct ParseSess {
     pub assume_incomplete_release: bool,
     /// Spans passed to `proc_macro::quote_span`. Each span has a numerical
     /// identifier represented by its position in the vector.
-    pub proc_macro_quoted_spans: Lock<Vec<Span>>,
+    pub proc_macro_quoted_spans: AppendOnlyVec<Span>,
     /// Used to generate new `AttrId`s. Every `AttrId` is unique.
     pub attr_id_generator: AttrIdGenerator,
 }
@@ -247,14 +248,14 @@ impl ParseSess {
             config: FxIndexSet::default(),
             check_config: CrateCheckConfig::default(),
             edition: ExpnId::root().expn_data().edition,
-            raw_identifier_spans: Lock::new(Vec::new()),
+            raw_identifier_spans: Default::default(),
             bad_unicode_identifiers: Lock::new(Default::default()),
             source_map,
             buffered_lints: Lock::new(vec![]),
             ambiguous_block_expr_parse: Lock::new(FxHashMap::default()),
             gated_spans: GatedSpans::default(),
             symbol_gallery: SymbolGallery::default(),
-            reached_eof: Lock::new(false),
+            reached_eof: AtomicBool::new(false),
             env_depinfo: Default::default(),
             file_depinfo: Default::default(),
             type_ascription_path_suggestions: Default::default(),
@@ -324,13 +325,13 @@ impl ParseSess {
     }
 
     pub fn save_proc_macro_span(&self, span: Span) -> usize {
-        let mut spans = self.proc_macro_quoted_spans.lock();
-        spans.push(span);
-        return spans.len() - 1;
+        self.proc_macro_quoted_spans.push(span)
     }
 
-    pub fn proc_macro_quoted_spans(&self) -> Vec<Span> {
-        self.proc_macro_quoted_spans.lock().clone()
+    pub fn proc_macro_quoted_spans(&self) -> impl Iterator<Item = (usize, Span)> + '_ {
+        // This is equivalent to `.iter().copied().enumerate()`, but that isn't possible for
+        // AppendOnlyVec, so we resort to this scheme.
+        self.proc_macro_quoted_spans.iter_enumerated()
     }
 
     #[track_caller]
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 12634f671..340bb158e 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -11,7 +11,7 @@ use crate::{filesearch, lint};
 pub use rustc_ast::attr::MarkedAttrs;
 pub use rustc_ast::Attribute;
 use rustc_data_structures::flock;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::jobserver::{self, Client};
 use rustc_data_structures::profiling::{duration_to_secs_str, SelfProfiler, SelfProfilerRef};
 use rustc_data_structures::sync::{
@@ -30,7 +30,7 @@ use rustc_macros::HashStable_Generic;
 pub use rustc_span::def_id::StableCrateId;
 use rustc_span::edition::Edition;
 use rustc_span::source_map::{FileLoader, RealFileLoader, SourceMap, Span};
-use rustc_span::{sym, SourceFileHashAlgorithm, Symbol};
+use rustc_span::{SourceFileHashAlgorithm, Symbol};
 use rustc_target::asm::InlineAsmArch;
 use rustc_target::spec::{CodeModel, PanicStrategy, RelocModel, RelroLevel};
 use rustc_target::spec::{
@@ -207,10 +207,10 @@ pub struct Session {
     pub asm_arch: Option<InlineAsmArch>,
 
     /// Set of enabled features for the current target.
-    pub target_features: FxHashSet<Symbol>,
+    pub target_features: FxIndexSet<Symbol>,
 
     /// Set of enabled features for the current target, including unstable ones.
-    pub unstable_target_features: FxHashSet<Symbol>,
+    pub unstable_target_features: FxIndexSet<Symbol>,
 }
 
 pub struct PerfStats {
@@ -224,6 +224,13 @@ pub struct PerfStats {
     pub normalize_projection_ty: AtomicUsize,
 }
 
+#[derive(PartialEq, Eq, PartialOrd, Ord)]
+pub enum MetadataKind {
+    None,
+    Uncompressed,
+    Compressed,
+}
+
 impl Session {
     pub fn miri_unleashed_feature(&self, span: Span, feature_gate: Option<Symbol>) {
         self.miri_unleashed_features.lock().push((span, feature_gate));
@@ -287,6 +294,43 @@ impl Session {
         self.crate_types.get().unwrap().as_slice()
     }
 
+    /// Returns true if the crate is a testing one.
+    pub fn is_test_crate(&self) -> bool {
+        self.opts.test
+    }
+
+    pub fn needs_crate_hash(&self) -> bool {
+        // Why is the crate hash needed for these configurations?
+        // - debug_assertions: for the "fingerprint the result" check in
+        //   `rustc_query_system::query::plumbing::execute_job`.
+        // - incremental: for query lookups.
+        // - needs_metadata: for putting into crate metadata.
+        // - instrument_coverage: for putting into coverage data (see
+        //   `hash_mir_source`).
+        cfg!(debug_assertions)
+            || self.opts.incremental.is_some()
+            || self.needs_metadata()
+            || self.instrument_coverage()
+    }
+
+    pub fn metadata_kind(&self) -> MetadataKind {
+        self.crate_types()
+            .iter()
+            .map(|ty| match *ty {
+                CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => {
+                    MetadataKind::None
+                }
+                CrateType::Rlib => MetadataKind::Uncompressed,
+                CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed,
+            })
+            .max()
+            .unwrap_or(MetadataKind::None)
+    }
+
+    pub fn needs_metadata(&self) -> bool {
+        self.metadata_kind() != MetadataKind::None
+    }
+
     pub fn init_crate_types(&self, crate_types: Vec<CrateType>) {
         self.crate_types.set(crate_types).expect("`crate_types` was initialized twice")
     }
@@ -964,40 +1008,6 @@ impl Session {
         || self.opts.unstable_opts.sanitizer.intersects(SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS | SanitizerSet::MEMORY | SanitizerSet::HWADDRESS)
     }
 
-    pub fn is_proc_macro_attr(&self, attr: &Attribute) -> bool {
-        [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive]
-            .iter()
-            .any(|kind| attr.has_name(*kind))
-    }
-
-    pub fn contains_name(&self, attrs: &[Attribute], name: Symbol) -> bool {
-        attrs.iter().any(|item| item.has_name(name))
-    }
-
-    pub fn find_by_name<'a>(
-        &'a self,
-        attrs: &'a [Attribute],
-        name: Symbol,
-    ) -> Option<&'a Attribute> {
-        attrs.iter().find(|attr| attr.has_name(name))
-    }
-
-    pub fn filter_by_name<'a>(
-        &'a self,
-        attrs: &'a [Attribute],
-        name: Symbol,
-    ) -> impl Iterator<Item = &'a Attribute> {
-        attrs.iter().filter(move |attr| attr.has_name(name))
-    }
-
-    pub fn first_attr_value_str_by_name(
-        &self,
-        attrs: &[Attribute],
-        name: Symbol,
-    ) -> Option<Symbol> {
-        attrs.iter().find(|at| at.has_name(name)).and_then(|at| at.value_str())
-    }
-
     pub fn diagnostic_width(&self) -> usize {
         let default_column_width = 140;
         if let Some(width) = self.opts.diagnostic_width {
@@ -1448,7 +1458,10 @@ pub fn build_session(
         CguReuseTracker::new_disabled()
     };
 
-    let prof = SelfProfilerRef::new(self_profiler, sopts.unstable_opts.time_passes);
+    let prof = SelfProfilerRef::new(
+        self_profiler,
+        sopts.unstable_opts.time_passes.then(|| sopts.unstable_opts.time_passes_format),
+    );
 
     let ctfe_backtrace = Lock::new(match env::var("RUSTC_CTFE_BACKTRACE") {
         Ok(ref val) if val == "immediate" => CtfeBacktrace::Immediate,
@@ -1488,8 +1501,8 @@ pub fn build_session(
         ctfe_backtrace,
         miri_unleashed_features: Lock::new(Default::default()),
         asm_arch,
-        target_features: FxHashSet::default(),
-        unstable_target_features: FxHashSet::default(),
+        target_features: Default::default(),
+        unstable_target_features: Default::default(),
     };
 
     validate_commandline_args_with_session_available(&sess);
diff --git a/compiler/rustc_session/src/utils.rs b/compiler/rustc_session/src/utils.rs
index b996d36a3..1d15e2c28 100644
--- a/compiler/rustc_session/src/utils.rs
+++ b/compiler/rustc_session/src/utils.rs
@@ -1,5 +1,6 @@
 use crate::session::Session;
 use rustc_data_structures::profiling::VerboseTimingGuard;
+use rustc_fs_util::try_canonicalize;
 use std::path::{Path, PathBuf};
 
 impl Session {
@@ -37,6 +38,10 @@ pub enum NativeLibKind {
     /// Argument which is passed to linker, relative order with libraries and other arguments
     /// is preserved
     LinkArg,
+
+    /// Module imported from WebAssembly
+    WasmImportModule,
+
     /// The library kind wasn't specified, `Dylib` is currently used as a default.
     Unspecified,
 }
@@ -50,7 +55,10 @@ impl NativeLibKind {
             NativeLibKind::Dylib { as_needed } | NativeLibKind::Framework { as_needed } => {
                 as_needed.is_some()
             }
-            NativeLibKind::RawDylib | NativeLibKind::Unspecified | NativeLibKind::LinkArg => false,
+            NativeLibKind::RawDylib
+            | NativeLibKind::Unspecified
+            | NativeLibKind::LinkArg
+            | NativeLibKind::WasmImportModule => false,
         }
     }
 
@@ -91,7 +99,7 @@ pub struct CanonicalizedPath {
 
 impl CanonicalizedPath {
     pub fn new(path: &Path) -> Self {
-        Self { original: path.to_owned(), canonicalized: std::fs::canonicalize(path).ok() }
+        Self { original: path.to_owned(), canonicalized: try_canonicalize(path).ok() }
     }
 
     pub fn canonicalized(&self) -> &PathBuf {
diff --git a/compiler/rustc_smir/Cargo.toml b/compiler/rustc_smir/Cargo.toml
index 5e0d1f369..fb97ee5be 100644
--- a/compiler/rustc_smir/Cargo.toml
+++ b/compiler/rustc_smir/Cargo.toml
@@ -4,25 +4,12 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-rustc_borrowck = { path = "../rustc_borrowck", optional = true }
-rustc_driver = { path = "../rustc_driver", optional = true }
-rustc_hir = { path = "../rustc_hir", optional = true }
-rustc_interface = { path = "../rustc_interface", optional = true }
 rustc_middle = { path = "../rustc_middle", optional = true }
-rustc_mir_dataflow = { path = "../rustc_mir_dataflow", optional = true }
-rustc_mir_transform = { path = "../rustc_mir_transform", optional = true }
-rustc_serialize = { path = "../rustc_serialize", optional = true }
-rustc_trait_selection = { path = "../rustc_trait_selection", optional = true }
+rustc_span = { path = "../rustc_span", optional = true }
+tracing = "0.1"
 
 [features]
 default = [
-    "rustc_borrowck",
-    "rustc_driver",
-    "rustc_hir",
-    "rustc_interface",
     "rustc_middle",
-    "rustc_mir_dataflow",
-    "rustc_mir_transform",
-    "rustc_serialize",
-    "rustc_trait_selection",
+    "rustc_span",
 ]
diff --git a/compiler/rustc_smir/README.md b/compiler/rustc_smir/README.md
index ae49098dd..31dee955f 100644
--- a/compiler/rustc_smir/README.md
+++ b/compiler/rustc_smir/README.md
@@ -73,3 +73,40 @@ git subtree pull --prefix=compiler/rustc_smir https://github.com/rust-lang/proje
 Note: only ever sync to rustc from the project-stable-mir's `smir` branch. Do not sync with your own forks.
 
 Then open a PR against rustc just like a regular PR.
+
+## Stable MIR Design
+
+The stable-mir will follow a similar approach to proc-macro2. It’s
+implementation will eventually be broken down into two main crates:
+
+- `stable_mir`: Public crate, to be published on crates.io, which will contain
+the stable data structure as well as proxy APIs to make calls to the
+compiler.
+- `rustc_smir`: The compiler crate that will translate from internal MIR to
+SMIR. This crate will also implement APIs that will be invoked by
+stable-mir to query the compiler for more information.
+
+This will help tools to communicate with the rust compiler via stable APIs. Tools will depend on
+`stable_mir` crate, which will invoke the compiler using APIs defined in `rustc_smir`. I.e.:
+
+```
+    ┌──────────────────────────────────┐           ┌──────────────────────────────────┐
+    │   External Tool     ┌──────────┐ │           │ ┌──────────┐   Rust Compiler     │
+    │                     │          │ │           │ │          │                     │
+    │                     │stable_mir| │           │ │rustc_smir│                     │
+    │                     │          │ ├──────────►| │          │                     │
+    │                     │          │ │◄──────────┤ │          │                     │
+    │                     │          │ │           │ │          │                     │
+    │                     │          │ │           │ │          │                     │
+    │                     └──────────┘ │           │ └──────────┘                     │
+    └──────────────────────────────────┘           └──────────────────────────────────┘
+```
+
+More details can be found here:
+https://hackmd.io/XhnYHKKuR6-LChhobvlT-g?view
+
+For now, the code for these two crates are in separate modules of this crate.
+The modules have the same name for simplicity. We also have a third module,
+`rustc_internal` which will expose APIs and definitions that allow users to
+gather information from internal MIR constructs that haven't been exposed in
+the `stable_mir` module.
diff --git a/compiler/rustc_smir/rust-toolchain.toml b/compiler/rustc_smir/rust-toolchain.toml
index 7b696fc1f..157dfd620 100644
--- a/compiler/rustc_smir/rust-toolchain.toml
+++ b/compiler/rustc_smir/rust-toolchain.toml
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-06-01"
+channel = "nightly-2023-02-28"
 components = [ "rustfmt", "rustc-dev" ]
diff --git a/compiler/rustc_smir/src/lib.rs b/compiler/rustc_smir/src/lib.rs
index 3e93c6bba..54d474db0 100644
--- a/compiler/rustc_smir/src/lib.rs
+++ b/compiler/rustc_smir/src/lib.rs
@@ -11,9 +11,9 @@
     test(attr(allow(unused_variables), deny(warnings)))
 )]
 #![cfg_attr(not(feature = "default"), feature(rustc_private))]
-#![deny(rustc::untranslatable_diagnostic)]
-#![deny(rustc::diagnostic_outside_of_impl)]
 
-pub mod mir;
+pub mod rustc_internal;
+pub mod stable_mir;
 
-pub mod very_unstable;
+// Make this module private for now since external users should not call these directly.
+mod rustc_smir;
diff --git a/compiler/rustc_smir/src/mir.rs b/compiler/rustc_smir/src/mir.rs
deleted file mode 100644
index 887e65729..000000000
--- a/compiler/rustc_smir/src/mir.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-pub use crate::very_unstable::hir::ImplicitSelfKind;
-pub use crate::very_unstable::middle::mir::{
-    visit::MutVisitor, AggregateKind, AssertKind, BasicBlock, BasicBlockData, BinOp, BindingForm,
-    BlockTailInfo, Body, BorrowKind, CastKind, ClearCrossCrate, Constant, ConstantKind,
-    CopyNonOverlapping, Coverage, FakeReadCause, Field, GeneratorInfo, InlineAsmOperand, Local,
-    LocalDecl, LocalInfo, LocalKind, Location, MirPhase, MirSource, NullOp, Operand, Place,
-    PlaceRef, ProjectionElem, ProjectionKind, Promoted, RetagKind, Rvalue, Safety, SourceInfo,
-    SourceScope, SourceScopeData, SourceScopeLocalData, Statement, StatementKind, UnOp,
-    UserTypeProjection, UserTypeProjections, VarBindingForm, VarDebugInfo, VarDebugInfoContents,
-};
diff --git a/compiler/rustc_smir/src/rustc_internal/mod.rs b/compiler/rustc_smir/src/rustc_internal/mod.rs
new file mode 100644
index 000000000..5998c8b65
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_internal/mod.rs
@@ -0,0 +1,32 @@
+//! Module that implements the bridge between Stable MIR and internal compiler MIR.
+//!
+//! For that, we define APIs that will temporarily be public to 3P that exposes rustc internal APIs
+//! until stable MIR is complete.
+
+use std::sync::RwLock;
+
+use crate::stable_mir;
+pub use rustc_span::def_id::{CrateNum, DefId};
+
+static DEF_ID_MAP: RwLock<Vec<DefId>> = RwLock::new(Vec::new());
+
+pub fn item_def_id(item: &stable_mir::CrateItem) -> DefId {
+    DEF_ID_MAP.read().unwrap()[item.0]
+}
+
+pub fn crate_item(did: DefId) -> stable_mir::CrateItem {
+    // FIXME: this becomes inefficient when we have too many ids
+    let mut map = DEF_ID_MAP.write().unwrap();
+    for (i, &d) in map.iter().enumerate() {
+        if d == did {
+            return stable_mir::CrateItem(i);
+        }
+    }
+    let id = map.len();
+    map.push(did);
+    stable_mir::CrateItem(id)
+}
+
+pub fn crate_num(item: &stable_mir::Crate) -> CrateNum {
+    item.id.into()
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs
new file mode 100644
index 000000000..4dad3c6bc
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_smir/mod.rs
@@ -0,0 +1,162 @@
+//! Module that implements what will become the rustc side of Stable MIR.
+//!
+//! This module is responsible for building Stable MIR components from internal components.
+//!
+//! This module is not intended to be invoked directly by users. It will eventually
+//! become the public API of rustc that will be invoked by the `stable_mir` crate.
+//!
+//! For now, we are developing everything inside `rustc`, thus, we keep this module private.
+
+use crate::{
+    rustc_internal::{crate_item, item_def_id},
+    stable_mir::{self},
+};
+use rustc_middle::ty::{tls::with, TyCtxt};
+use rustc_span::def_id::{CrateNum, LOCAL_CRATE};
+use tracing::debug;
+
+/// Get information about the local crate.
+pub fn local_crate() -> stable_mir::Crate {
+    with(|tcx| smir_crate(tcx, LOCAL_CRATE))
+}
+
+/// Retrieve a list of all external crates.
+pub fn external_crates() -> Vec<stable_mir::Crate> {
+    with(|tcx| tcx.crates(()).iter().map(|crate_num| smir_crate(tcx, *crate_num)).collect())
+}
+
+/// Find a crate with the given name.
+pub fn find_crate(name: &str) -> Option<stable_mir::Crate> {
+    with(|tcx| {
+        [LOCAL_CRATE].iter().chain(tcx.crates(()).iter()).find_map(|crate_num| {
+            let crate_name = tcx.crate_name(*crate_num).to_string();
+            (name == crate_name).then(|| smir_crate(tcx, *crate_num))
+        })
+    })
+}
+
+/// Retrieve all items of the local crate that have a MIR associated with them.
+pub fn all_local_items() -> stable_mir::CrateItems {
+    with(|tcx| tcx.mir_keys(()).iter().map(|item| crate_item(item.to_def_id())).collect())
+}
+
+pub fn entry_fn() -> Option<stable_mir::CrateItem> {
+    with(|tcx| Some(crate_item(tcx.entry_fn(())?.0)))
+}
+
+/// Build a stable mir crate from a given crate number.
+fn smir_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> stable_mir::Crate {
+    let crate_name = tcx.crate_name(crate_num).to_string();
+    let is_local = crate_num == LOCAL_CRATE;
+    debug!(?crate_name, ?crate_num, "smir_crate");
+    stable_mir::Crate { id: crate_num.into(), name: crate_name, is_local }
+}
+
+pub fn mir_body(item: &stable_mir::CrateItem) -> stable_mir::mir::Body {
+    with(|tcx| {
+        let def_id = item_def_id(item);
+        let mir = tcx.optimized_mir(def_id);
+        stable_mir::mir::Body {
+            blocks: mir
+                .basic_blocks
+                .iter()
+                .map(|block| stable_mir::mir::BasicBlock {
+                    terminator: rustc_terminator_to_terminator(block.terminator()),
+                    statements: block.statements.iter().map(rustc_statement_to_statement).collect(),
+                })
+                .collect(),
+        }
+    })
+}
+
+fn rustc_statement_to_statement(
+    s: &rustc_middle::mir::Statement<'_>,
+) -> stable_mir::mir::Statement {
+    use rustc_middle::mir::StatementKind::*;
+    match &s.kind {
+        Assign(assign) => stable_mir::mir::Statement::Assign(
+            rustc_place_to_place(&assign.0),
+            rustc_rvalue_to_rvalue(&assign.1),
+        ),
+        FakeRead(_) => todo!(),
+        SetDiscriminant { .. } => todo!(),
+        Deinit(_) => todo!(),
+        StorageLive(_) => todo!(),
+        StorageDead(_) => todo!(),
+        Retag(_, _) => todo!(),
+        PlaceMention(_) => todo!(),
+        AscribeUserType(_, _) => todo!(),
+        Coverage(_) => todo!(),
+        Intrinsic(_) => todo!(),
+        ConstEvalCounter => todo!(),
+        Nop => stable_mir::mir::Statement::Nop,
+    }
+}
+
+fn rustc_rvalue_to_rvalue(rvalue: &rustc_middle::mir::Rvalue<'_>) -> stable_mir::mir::Operand {
+    use rustc_middle::mir::Rvalue::*;
+    match rvalue {
+        Use(op) => rustc_op_to_op(op),
+        Repeat(_, _) => todo!(),
+        Ref(_, _, _) => todo!(),
+        ThreadLocalRef(_) => todo!(),
+        AddressOf(_, _) => todo!(),
+        Len(_) => todo!(),
+        Cast(_, _, _) => todo!(),
+        BinaryOp(_, _) => todo!(),
+        CheckedBinaryOp(_, _) => todo!(),
+        NullaryOp(_, _) => todo!(),
+        UnaryOp(_, _) => todo!(),
+        Discriminant(_) => todo!(),
+        Aggregate(_, _) => todo!(),
+        ShallowInitBox(_, _) => todo!(),
+        CopyForDeref(_) => todo!(),
+    }
+}
+
+fn rustc_op_to_op(op: &rustc_middle::mir::Operand<'_>) -> stable_mir::mir::Operand {
+    use rustc_middle::mir::Operand::*;
+    match op {
+        Copy(place) => stable_mir::mir::Operand::Copy(rustc_place_to_place(place)),
+        Move(place) => stable_mir::mir::Operand::Move(rustc_place_to_place(place)),
+        Constant(c) => stable_mir::mir::Operand::Constant(c.to_string()),
+    }
+}
+
+fn rustc_place_to_place(place: &rustc_middle::mir::Place<'_>) -> stable_mir::mir::Place {
+    assert_eq!(&place.projection[..], &[]);
+    stable_mir::mir::Place { local: place.local.as_usize() }
+}
+
+fn rustc_terminator_to_terminator(
+    terminator: &rustc_middle::mir::Terminator<'_>,
+) -> stable_mir::mir::Terminator {
+    use rustc_middle::mir::TerminatorKind::*;
+    use stable_mir::mir::Terminator;
+    match &terminator.kind {
+        Goto { target } => Terminator::Goto { target: target.as_usize() },
+        SwitchInt { discr, targets } => Terminator::SwitchInt {
+            discr: rustc_op_to_op(discr),
+            targets: targets
+                .iter()
+                .map(|(value, target)| stable_mir::mir::SwitchTarget {
+                    value,
+                    target: target.as_usize(),
+                })
+                .collect(),
+            otherwise: targets.otherwise().as_usize(),
+        },
+        Resume => Terminator::Resume,
+        Terminate => Terminator::Abort,
+        Return => Terminator::Return,
+        Unreachable => Terminator::Unreachable,
+        Drop { .. } => todo!(),
+        Call { .. } => todo!(),
+        Assert { .. } => todo!(),
+        Yield { .. } => todo!(),
+        GeneratorDrop => todo!(),
+        FalseEdge { .. } => todo!(),
+        FalseUnwind { .. } => todo!(),
+        InlineAsm { .. } => todo!(),
+    }
+}
diff --git a/compiler/rustc_smir/src/stable_mir/mir.rs b/compiler/rustc_smir/src/stable_mir/mir.rs
new file mode 100644
index 000000000..a9dbc3463
--- /dev/null
+++ b/compiler/rustc_smir/src/stable_mir/mir.rs
@@ -0,0 +1,3 @@
+mod body;
+
+pub use body::*;
diff --git a/compiler/rustc_smir/src/stable_mir/mir/body.rs b/compiler/rustc_smir/src/stable_mir/mir/body.rs
new file mode 100644
index 000000000..c504065c9
--- /dev/null
+++ b/compiler/rustc_smir/src/stable_mir/mir/body.rs
@@ -0,0 +1,69 @@
+#[derive(Clone, Debug)]
+pub struct Body {
+    pub blocks: Vec<BasicBlock>,
+}
+
+#[derive(Clone, Debug)]
+pub struct BasicBlock {
+    pub statements: Vec<Statement>,
+    pub terminator: Terminator,
+}
+
+#[derive(Clone, Debug)]
+pub enum Terminator {
+    Goto {
+        target: usize,
+    },
+    SwitchInt {
+        discr: Operand,
+        targets: Vec<SwitchTarget>,
+        otherwise: usize,
+    },
+    Resume,
+    Abort,
+    Return,
+    Unreachable,
+    Drop {
+        place: Place,
+        target: usize,
+        unwind: Option<usize>,
+    },
+    Call {
+        func: Operand,
+        args: Vec<Operand>,
+        destination: Place,
+        target: Option<usize>,
+        cleanup: Option<usize>,
+    },
+    Assert {
+        cond: Operand,
+        expected: bool,
+        msg: String,
+        target: usize,
+        cleanup: Option<usize>,
+    },
+}
+
+#[derive(Clone, Debug)]
+pub enum Statement {
+    Assign(Place, Operand),
+    Nop,
+}
+
+#[derive(Clone, Debug)]
+pub enum Operand {
+    Copy(Place),
+    Move(Place),
+    Constant(String),
+}
+
+#[derive(Clone, Debug)]
+pub struct Place {
+    pub local: usize,
+}
+
+#[derive(Clone, Debug)]
+pub struct SwitchTarget {
+    pub value: u128,
+    pub target: usize,
+}
diff --git a/compiler/rustc_smir/src/stable_mir/mod.rs b/compiler/rustc_smir/src/stable_mir/mod.rs
new file mode 100644
index 000000000..1d2efb5ea
--- /dev/null
+++ b/compiler/rustc_smir/src/stable_mir/mod.rs
@@ -0,0 +1,73 @@
+//! Module that implements the public interface to the Stable MIR.
+//!
+//! This module shall contain all type definitions and APIs that we expect 3P tools to invoke to
+//! interact with the compiler.
+//!
+//! The goal is to eventually move this module to its own crate which shall be published on
+//! [crates.io](https://crates.io).
+//!
+//! ## Note:
+//!
+//! There shouldn't be any direct references to internal compiler constructs in this module.
+//! If you need an internal construct, consider using `rustc_internal` or `rustc_smir`.
+
+pub mod mir;
+
+/// Use String for now but we should replace it.
+pub type Symbol = String;
+
+/// The number that identifies a crate.
+pub type CrateNum = usize;
+
+/// A unique identification number for each item accessible for the current compilation unit.
+pub type DefId = usize;
+
+/// A list of crate items.
+pub type CrateItems = Vec<CrateItem>;
+
+/// Holds information about a crate.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Crate {
+    pub(crate) id: CrateNum,
+    pub name: Symbol,
+    pub is_local: bool,
+}
+
+/// Holds information about an item in the crate.
+/// For now, it only stores the item DefId. Use functions inside `rustc_internal` module to
+/// use this item.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct CrateItem(pub(crate) DefId);
+
+impl CrateItem {
+    pub fn body(&self) -> mir::Body {
+        crate::rustc_smir::mir_body(self)
+    }
+}
+
+/// Return the function where execution starts if the current
+/// crate defines that. This is usually `main`, but could be
+/// `start` if the crate is a no-std crate.
+pub fn entry_fn() -> Option<CrateItem> {
+    crate::rustc_smir::entry_fn()
+}
+
+/// Access to the local crate.
+pub fn local_crate() -> Crate {
+    crate::rustc_smir::local_crate()
+}
+
+/// Try to find a crate with the given name.
+pub fn find_crate(name: &str) -> Option<Crate> {
+    crate::rustc_smir::find_crate(name)
+}
+
+/// Try to find a crate with the given name.
+pub fn external_crates() -> Vec<Crate> {
+    crate::rustc_smir::external_crates()
+}
+
+/// Retrieve all items in the local crate that have a MIR associated with them.
+pub fn all_local_items() -> CrateItems {
+    crate::rustc_smir::all_local_items()
+}
diff --git a/compiler/rustc_smir/src/very_unstable.rs b/compiler/rustc_smir/src/very_unstable.rs
deleted file mode 100644
index 12ba133db..000000000
--- a/compiler/rustc_smir/src/very_unstable.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-//! This module reexports various crates and modules from unstable rustc APIs.
-//! Add anything you need here and it will get slowly transferred to a stable API.
-//! Only use rustc_smir in your dependencies and use the reexports here instead of
-//! directly referring to the unstable crates.
-
-macro_rules! crates {
-    ($($rustc_name:ident -> $name:ident,)*) => {
-        $(
-            #[cfg(not(feature = "default"))]
-            pub extern crate $rustc_name as $name;
-            #[cfg(feature = "default")]
-            pub use $rustc_name as $name;
-        )*
-    }
-}
-
-crates! {
-    rustc_borrowck -> borrowck,
-    rustc_driver -> driver,
-    rustc_hir -> hir,
-    rustc_interface -> interface,
-    rustc_middle -> middle,
-    rustc_mir_dataflow -> dataflow,
-    rustc_mir_transform -> transform,
-    rustc_serialize -> serialize,
-    rustc_trait_selection -> trait_selection,
-}
diff --git a/compiler/rustc_span/Cargo.toml b/compiler/rustc_span/Cargo.toml
index ae81d95e2..a7c7575f3 100644
--- a/compiler/rustc_span/Cargo.toml
+++ b/compiler/rustc_span/Cargo.toml
@@ -18,3 +18,4 @@ tracing = "0.1"
 sha1 = "0.10.0"
 sha2 = "0.10.1"
 md5 = { package = "md-5", version = "0.10.0" }
+indexmap = { version = "1.9.3" }
diff --git a/compiler/rustc_span/src/def_id.rs b/compiler/rustc_span/src/def_id.rs
index 162c15574..b2c58caff 100644
--- a/compiler/rustc_span/src/def_id.rs
+++ b/compiler/rustc_span/src/def_id.rs
@@ -1,13 +1,17 @@
 use crate::{HashStableContext, Symbol};
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey};
+use rustc_data_structures::unhash::Unhasher;
 use rustc_data_structures::AtomicRef;
 use rustc_index::vec::Idx;
 use rustc_macros::HashStable_Generic;
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use std::borrow::Borrow;
 use std::fmt;
-use std::hash::{Hash, Hasher};
+use std::hash::{BuildHasherDefault, Hash, Hasher};
+
+pub type StableCrateIdMap =
+    indexmap::IndexMap<StableCrateId, CrateNum, BuildHasherDefault<Unhasher>>;
 
 rustc_index::newtype_index! {
     #[custom_encodable]
diff --git a/compiler/rustc_span/src/edit_distance.rs b/compiler/rustc_span/src/edit_distance.rs
index 89f0386e3..259f42386 100644
--- a/compiler/rustc_span/src/edit_distance.rs
+++ b/compiler/rustc_span/src/edit_distance.rs
@@ -174,10 +174,10 @@ pub fn find_best_match_for_name(
 fn find_best_match_for_name_impl(
     use_substring_score: bool,
     candidates: &[Symbol],
-    lookup: Symbol,
+    lookup_symbol: Symbol,
     dist: Option<usize>,
 ) -> Option<Symbol> {
-    let lookup = lookup.as_str();
+    let lookup = lookup_symbol.as_str();
     let lookup_uppercase = lookup.to_uppercase();
 
     // Priority of matches:
@@ -190,6 +190,8 @@ fn find_best_match_for_name_impl(
 
     let mut dist = dist.unwrap_or_else(|| cmp::max(lookup.len(), 3) / 3);
     let mut best = None;
+    // store the candidates with the same distance, only for `use_substring_score` current.
+    let mut next_candidates = vec![];
     for c in candidates {
         match if use_substring_score {
             edit_distance_with_substrings(lookup, c.as_str(), dist)
@@ -198,12 +200,36 @@ fn find_best_match_for_name_impl(
         } {
             Some(0) => return Some(*c),
             Some(d) => {
-                dist = d - 1;
+                if use_substring_score {
+                    if d < dist {
+                        dist = d;
+                        next_candidates.clear();
+                    } else {
+                        // `d == dist` here, we need to store the candidates with the same distance
+                        // so we won't decrease the distance in the next loop.
+                    }
+                    next_candidates.push(*c);
+                } else {
+                    dist = d - 1;
+                }
                 best = Some(*c);
             }
             None => {}
         }
     }
+
+    // We have a tie among several candidates, try to select the best among them ignoring substrings.
+    // For example, the candidates list `force_capture`, `capture`, and user inputted `forced_capture`,
+    // we select `force_capture` with a extra round of edit distance calculation.
+    if next_candidates.len() > 1 {
+        debug_assert!(use_substring_score);
+        best = find_best_match_for_name_impl(
+            false,
+            &next_candidates,
+            lookup_symbol,
+            Some(lookup.len()),
+        );
+    }
     if best.is_some() {
         return best;
     }
diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs
index dee823eef..08c441403 100644
--- a/compiler/rustc_span/src/hygiene.rs
+++ b/compiler/rustc_span/src/hygiene.rs
@@ -109,7 +109,7 @@ fn assert_default_hashing_controls<CTX: HashStableContext>(ctx: &CTX, msg: &str)
         // This is the case for instance when building a hash for name mangling.
         // Such configuration must not be used for metadata.
         HashingControls { hash_spans }
-            if hash_spans == !ctx.unstable_opts_incremental_ignore_spans() => {}
+            if hash_spans != ctx.unstable_opts_incremental_ignore_spans() => {}
         other => panic!("Attempted hashing of {msg} with non-default HashingControls: {other:?}"),
     }
 }
@@ -880,7 +880,7 @@ impl Span {
     pub fn fresh_expansion(self, expn_id: LocalExpnId) -> Span {
         HygieneData::with(|data| {
             self.with_ctxt(data.apply_mark(
-                SyntaxContext::root(),
+                self.ctxt(),
                 expn_id.to_expn_id(),
                 Transparency::Transparent,
             ))
@@ -1151,6 +1151,7 @@ pub enum DesugaringKind {
     Await,
     ForLoop,
     WhileLoop,
+    Replace,
 }
 
 impl DesugaringKind {
@@ -1166,6 +1167,7 @@ impl DesugaringKind {
             DesugaringKind::OpaqueTy => "`impl Trait`",
             DesugaringKind::ForLoop => "`for` loop",
             DesugaringKind::WhileLoop => "`while` loop",
+            DesugaringKind::Replace => "drop and replace",
         }
     }
 }
@@ -1205,7 +1207,7 @@ impl HygieneEncodeContext {
         // a `SyntaxContext` that we haven't seen before
         while !self.latest_ctxts.lock().is_empty() || !self.latest_expns.lock().is_empty() {
             debug!(
-                "encode_hygiene: Serializing a round of {:?} SyntaxContextDatas: {:?}",
+                "encode_hygiene: Serializing a round of {:?} SyntaxContextData: {:?}",
                 self.latest_ctxts.lock().len(),
                 self.latest_ctxts
             );
diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs
index 873cd33f6..28a8d8fc1 100644
--- a/compiler/rustc_span/src/lib.rs
+++ b/compiler/rustc_span/src/lib.rs
@@ -20,6 +20,7 @@
 #![feature(min_specialization)]
 #![feature(rustc_attrs)]
 #![feature(let_chains)]
+#![feature(round_char_boundary)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -87,6 +88,14 @@ pub struct SessionGlobals {
     symbol_interner: symbol::Interner,
     span_interner: Lock<span_encoding::SpanInterner>,
     hygiene_data: Lock<hygiene::HygieneData>,
+
+    /// A reference to the source map in the `Session`. It's an `Option`
+    /// because it can't be initialized until `Session` is created, which
+    /// happens after `SessionGlobals`. `set_source_map` does the
+    /// initialization.
+    ///
+    /// This field should only be used in places where the `Session` is truly
+    /// not available, such as `<Span as Debug>::fmt`.
     source_map: Lock<Option<Lrc<SourceMap>>>,
 }
 
@@ -795,6 +804,18 @@ impl Span {
         })
     }
 
+    /// Splits a span into two composite spans around a certain position.
+    pub fn split_at(self, pos: u32) -> (Span, Span) {
+        let len = self.hi().0 - self.lo().0;
+        debug_assert!(pos <= len);
+
+        let split_pos = BytePos(self.lo().0 + pos);
+        (
+            Span::new(self.lo(), split_pos, self.ctxt(), self.parent()),
+            Span::new(split_pos, self.hi(), self.ctxt(), self.parent()),
+        )
+    }
+
     /// Returns a `Span` that would enclose both `self` and `end`.
     ///
     /// Note that this can also be used to extend the span "backwards":
@@ -1001,16 +1022,9 @@ impl<D: Decoder> Decodable<D> for Span {
     }
 }
 
-/// Calls the provided closure, using the provided `SourceMap` to format
-/// any spans that are debug-printed during the closure's execution.
-///
-/// Normally, the global `TyCtxt` is used to retrieve the `SourceMap`
-/// (see `rustc_interface::callbacks::span_debug1`). However, some parts
-/// of the compiler (e.g. `rustc_parse`) may debug-print `Span`s before
-/// a `TyCtxt` is available. In this case, we fall back to
-/// the `SourceMap` provided to this function. If that is not available,
-/// we fall back to printing the raw `Span` field values.
-pub fn with_source_map<T, F: FnOnce() -> T>(source_map: Lrc<SourceMap>, f: F) -> T {
+/// Insert `source_map` into the session globals for the duration of the
+/// closure's execution.
+pub fn set_source_map<T, F: FnOnce() -> T>(source_map: Lrc<SourceMap>, f: F) -> T {
     with_session_globals(|session_globals| {
         *session_globals.source_map.borrow_mut() = Some(source_map);
     });
@@ -1029,6 +1043,8 @@ pub fn with_source_map<T, F: FnOnce() -> T>(source_map: Lrc<SourceMap>, f: F) ->
 
 impl fmt::Debug for Span {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Use the global `SourceMap` to print the span. If that's not
+        // available, fall back to printing the raw values.
         with_session_globals(|session_globals| {
             if let Some(source_map) = &*session_globals.source_map.borrow() {
                 write!(f, "{} ({:?})", source_map.span_to_diagnostic_string(*self), self.ctxt())
@@ -1303,7 +1319,6 @@ pub struct SourceFileDiffs {
 }
 
 /// A single source in the [`SourceMap`].
-#[derive(Clone)]
 pub struct SourceFile {
     /// The name of the file that the source came from. Source that doesn't
     /// originate from files has names between angle brackets by convention
@@ -1334,6 +1349,25 @@ pub struct SourceFile {
     pub cnum: CrateNum,
 }
 
+impl Clone for SourceFile {
+    fn clone(&self) -> Self {
+        Self {
+            name: self.name.clone(),
+            src: self.src.clone(),
+            src_hash: self.src_hash,
+            external_src: Lock::new(self.external_src.borrow().clone()),
+            start_pos: self.start_pos,
+            end_pos: self.end_pos,
+            lines: Lock::new(self.lines.borrow().clone()),
+            multibyte_chars: self.multibyte_chars.clone(),
+            non_narrow_chars: self.non_narrow_chars.clone(),
+            normalized_pos: self.normalized_pos.clone(),
+            name_hash: self.name_hash,
+            cnum: self.cnum,
+        }
+    }
+}
+
 impl<S: Encoder> Encodable<S> for SourceFile {
     fn encode(&self, s: &mut S) {
         self.name.encode(s);
@@ -2018,13 +2052,13 @@ pub type FileLinesResult = Result<FileLines, SpanLinesError>;
 
 #[derive(Clone, PartialEq, Eq, Debug)]
 pub enum SpanLinesError {
-    DistinctSources(DistinctSources),
+    DistinctSources(Box<DistinctSources>),
 }
 
 #[derive(Clone, PartialEq, Eq, Debug)]
 pub enum SpanSnippetError {
     IllFormedSpan(Span),
-    DistinctSources(DistinctSources),
+    DistinctSources(Box<DistinctSources>),
     MalformedForSourcemap(MalformedSourceMapPositions),
     SourceNotAvailable { filename: FileName },
 }
diff --git a/compiler/rustc_span/src/profiling.rs b/compiler/rustc_span/src/profiling.rs
index 0ab890b9f..66e5369da 100644
--- a/compiler/rustc_span/src/profiling.rs
+++ b/compiler/rustc_span/src/profiling.rs
@@ -1,3 +1,5 @@
+use crate::source_map::SourceMap;
+
 use std::borrow::Borrow;
 
 use rustc_data_structures::profiling::EventArgRecorder;
@@ -11,25 +13,17 @@ pub trait SpannedEventArgRecorder {
     ///
     /// Note: when self-profiling with costly event arguments, at least one argument
     /// needs to be recorded. A panic will be triggered if that doesn't happen.
-    fn record_arg_with_span<A>(&mut self, event_arg: A, span: crate::Span)
+    fn record_arg_with_span<A>(&mut self, source_map: &SourceMap, event_arg: A, span: crate::Span)
     where
         A: Borrow<str> + Into<String>;
 }
 
 impl SpannedEventArgRecorder for EventArgRecorder<'_> {
-    fn record_arg_with_span<A>(&mut self, event_arg: A, span: crate::Span)
+    fn record_arg_with_span<A>(&mut self, source_map: &SourceMap, event_arg: A, span: crate::Span)
     where
         A: Borrow<str> + Into<String>,
     {
         self.record_arg(event_arg);
-
-        let span_arg = crate::with_session_globals(|session_globals| {
-            if let Some(source_map) = &*session_globals.source_map.borrow() {
-                source_map.span_to_embeddable_string(span)
-            } else {
-                format!("{span:?}")
-            }
-        });
-        self.record_arg(span_arg);
+        self.record_arg(source_map.span_to_embeddable_string(span));
     }
 }
diff --git a/compiler/rustc_span/src/source_map.rs b/compiler/rustc_span/src/source_map.rs
index 2e339a9d2..56573814e 100644
--- a/compiler/rustc_span/src/source_map.rs
+++ b/compiler/rustc_span/src/source_map.rs
@@ -100,6 +100,9 @@ pub trait FileLoader {
 
     /// Read the contents of a UTF-8 file into memory.
     fn read_file(&self, path: &Path) -> io::Result<String>;
+
+    /// Read the contents of a potentially non-UTF-8 file into memory.
+    fn read_binary_file(&self, path: &Path) -> io::Result<Vec<u8>>;
 }
 
 /// A FileLoader that uses std::fs to load real files.
@@ -113,6 +116,10 @@ impl FileLoader for RealFileLoader {
     fn read_file(&self, path: &Path) -> io::Result<String> {
         fs::read_to_string(path)
     }
+
+    fn read_binary_file(&self, path: &Path) -> io::Result<Vec<u8>> {
+        fs::read(path)
+    }
 }
 
 /// This is a [SourceFile] identifier that is used to correlate source files between
@@ -220,9 +227,7 @@ impl SourceMap {
     /// Unlike `load_file`, guarantees that no normalization like BOM-removal
     /// takes place.
     pub fn load_binary_file(&self, path: &Path) -> io::Result<Vec<u8>> {
-        // Ideally, this should use `self.file_loader`, but it can't
-        // deal with binary files yet.
-        let bytes = fs::read(path)?;
+        let bytes = self.file_loader.read_binary_file(path)?;
 
         // We need to add file to the `SourceMap`, so that it is present
         // in dep-info. There's also an edge case that file might be both
@@ -443,25 +448,36 @@ impl SourceMap {
         sp: Span,
         filename_display_pref: FileNameDisplayPreference,
     ) -> String {
-        if self.files.borrow().source_files.is_empty() || sp.is_dummy() {
-            return "no-location".to_string();
-        }
+        let (source_file, lo_line, lo_col, hi_line, hi_col) = self.span_to_location_info(sp);
+
+        let file_name = match source_file {
+            Some(sf) => sf.name.display(filename_display_pref).to_string(),
+            None => return "no-location".to_string(),
+        };
 
-        let lo = self.lookup_char_pos(sp.lo());
-        let hi = self.lookup_char_pos(sp.hi());
         format!(
-            "{}:{}:{}{}",
-            lo.file.name.display(filename_display_pref),
-            lo.line,
-            lo.col.to_usize() + 1,
+            "{file_name}:{lo_line}:{lo_col}{}",
             if let FileNameDisplayPreference::Short = filename_display_pref {
                 String::new()
             } else {
-                format!(": {}:{}", hi.line, hi.col.to_usize() + 1)
+                format!(": {hi_line}:{hi_col}")
             }
         )
     }
 
+    pub fn span_to_location_info(
+        &self,
+        sp: Span,
+    ) -> (Option<Lrc<SourceFile>>, usize, usize, usize, usize) {
+        if self.files.borrow().source_files.is_empty() || sp.is_dummy() {
+            return (None, 0, 0, 0, 0);
+        }
+
+        let lo = self.lookup_char_pos(sp.lo());
+        let hi = self.lookup_char_pos(sp.hi());
+        (Some(lo.file), lo.line, lo.col.to_usize() + 1, hi.line, hi.col.to_usize() + 1)
+    }
+
     /// Format the span location suitable for embedding in build artifacts
     pub fn span_to_embeddable_string(&self, sp: Span) -> String {
         self.span_to_string(sp, FileNameDisplayPreference::Remapped)
@@ -526,10 +542,10 @@ impl SourceMap {
         let hi = self.lookup_char_pos(sp.hi());
         trace!(?hi);
         if lo.file.start_pos != hi.file.start_pos {
-            return Err(SpanLinesError::DistinctSources(DistinctSources {
+            return Err(SpanLinesError::DistinctSources(Box::new(DistinctSources {
                 begin: (lo.file.name.clone(), lo.file.start_pos),
                 end: (hi.file.name.clone(), hi.file.start_pos),
-            }));
+            })));
         }
         Ok((lo, hi))
     }
@@ -587,10 +603,10 @@ impl SourceMap {
         let local_end = self.lookup_byte_offset(sp.hi());
 
         if local_begin.sf.start_pos != local_end.sf.start_pos {
-            Err(SpanSnippetError::DistinctSources(DistinctSources {
+            Err(SpanSnippetError::DistinctSources(Box::new(DistinctSources {
                 begin: (local_begin.sf.name.clone(), local_begin.sf.start_pos),
                 end: (local_end.sf.name.clone(), local_end.sf.start_pos),
-            }))
+            })))
         } else {
             self.ensure_source_file_source_present(local_begin.sf.clone());
 
@@ -1003,36 +1019,19 @@ impl SourceMap {
 
         let src = local_begin.sf.external_src.borrow();
 
-        // We need to extend the snippet to the end of the src rather than to end_index so when
-        // searching forwards for boundaries we've got somewhere to search.
-        let snippet = if let Some(ref src) = local_begin.sf.src {
-            &src[start_index..]
+        let snippet = if let Some(src) = &local_begin.sf.src {
+            src
         } else if let Some(src) = src.get_source() {
-            &src[start_index..]
+            src
         } else {
             return 1;
         };
-        debug!("snippet=`{:?}`", snippet);
-
-        let mut target = if forwards { end_index + 1 } else { end_index - 1 };
-        debug!("initial target=`{:?}`", target);
 
-        while !snippet.is_char_boundary(target - start_index) && target < source_len {
-            target = if forwards {
-                target + 1
-            } else {
-                match target.checked_sub(1) {
-                    Some(target) => target,
-                    None => {
-                        break;
-                    }
-                }
-            };
-            debug!("target=`{:?}`", target);
+        if forwards {
+            (snippet.ceil_char_boundary(end_index + 1) - end_index) as u32
+        } else {
+            (end_index - snippet.floor_char_boundary(end_index - 1)) as u32
         }
-        debug!("final target=`{:?}`", target);
-
-        if forwards { (target - end_index) as u32 } else { (end_index - target) as u32 }
     }
 
     pub fn get_source_file(&self, filename: &FileName) -> Option<Lrc<SourceFile>> {
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 6272bf7f2..6bfae3771 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -357,6 +357,7 @@ symbols! {
         always,
         and,
         and_then,
+        anon,
         anonymous_lifetime_in_impl_trait,
         any,
         append_const_msg,
@@ -429,6 +430,7 @@ symbols! {
         borrowck_graphviz_format,
         borrowck_graphviz_postflow,
         box_free,
+        box_new,
         box_patterns,
         box_syntax,
         bpf_target_feature,
@@ -721,6 +723,8 @@ symbols! {
         fn_mut,
         fn_once,
         fn_once_output,
+        fn_ptr_addr,
+        fn_ptr_trait,
         forbid,
         forget,
         format,
@@ -791,13 +795,13 @@ symbols! {
         i64,
         i8,
         ident,
-        identity_future,
         if_let,
         if_let_guard,
         if_while_or_patterns,
         ignore,
         impl_header_lifetime_elision,
         impl_lint_pass,
+        impl_trait_in_assoc_type,
         impl_trait_in_bindings,
         impl_trait_in_fn_trait_return,
         impl_trait_projections,
@@ -984,6 +988,7 @@ symbols! {
         never_type_fallback,
         new,
         new_binary,
+        new_const,
         new_debug,
         new_display,
         new_lower_exp,
@@ -1043,6 +1048,7 @@ symbols! {
         optin_builtin_traits,
         option,
         option_env,
+        option_payload_ptr,
         options,
         or,
         or_patterns,
@@ -1065,6 +1071,7 @@ symbols! {
         panic_implementation,
         panic_info,
         panic_location,
+        panic_misaligned_pointer_dereference,
         panic_nounwind,
         panic_runtime,
         panic_str,
@@ -1153,6 +1160,7 @@ symbols! {
         read_enum_variant_arg,
         read_struct,
         read_struct_field,
+        read_via_copy,
         readonly,
         realloc,
         reason,
@@ -1167,7 +1175,9 @@ symbols! {
         reg32,
         reg64,
         reg_abcd,
+        reg_addr,
         reg_byte,
+        reg_data,
         reg_iw,
         reg_nonzero,
         reg_pair,
@@ -1190,6 +1200,7 @@ symbols! {
         residual,
         result,
         return_position_impl_trait_in_trait,
+        return_type_notation,
         rhs,
         rintf32,
         rintf64,
@@ -1197,6 +1208,8 @@ symbols! {
         rlib,
         rotate_left,
         rotate_right,
+        roundevenf32,
+        roundevenf64,
         roundf32,
         roundf64,
         rt,
@@ -1238,6 +1251,7 @@ symbols! {
         rustc_diagnostic_macros,
         rustc_dirty,
         rustc_do_not_const_check,
+        rustc_doc_primitive,
         rustc_dummy,
         rustc_dump_env_program_clauses,
         rustc_dump_program_clauses,
@@ -1486,6 +1500,7 @@ symbols! {
         trait_alias,
         trait_upcasting,
         transmute,
+        transmute_generic_consts,
         transmute_opts,
         transmute_trait,
         transparent,
diff --git a/compiler/rustc_symbol_mangling/locales/en-US.ftl b/compiler/rustc_symbol_mangling/messages.ftl
index b7d48280f..b7d48280f 100644
--- a/compiler/rustc_symbol_mangling/locales/en-US.ftl
+++ b/compiler/rustc_symbol_mangling/messages.ftl
diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs
index 2368468c8..5cbca8192 100644
--- a/compiler/rustc_symbol_mangling/src/legacy.rs
+++ b/compiler/rustc_symbol_mangling/src/legacy.rs
@@ -65,6 +65,10 @@ pub(super) fn mangle<'tcx>(
         )
         .unwrap();
 
+    if let ty::InstanceDef::ThreadLocalShim(..) = instance.def {
+        let _ = printer.write_str("{{tls-shim}}");
+    }
+
     if let ty::InstanceDef::VTableShim(..) = instance.def {
         let _ = printer.write_str("{{vtable-shim}}");
     }
diff --git a/compiler/rustc_symbol_mangling/src/lib.rs b/compiler/rustc_symbol_mangling/src/lib.rs
index d9ce73734..c2fd3304f 100644
--- a/compiler/rustc_symbol_mangling/src/lib.rs
+++ b/compiler/rustc_symbol_mangling/src/lib.rs
@@ -119,7 +119,7 @@ pub mod errors;
 pub mod test;
 pub mod typeid;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 /// This function computes the symbol name for the given `instance` and the
 /// given instantiating crate. That is, if you know that instance X is
diff --git a/compiler/rustc_symbol_mangling/src/test.rs b/compiler/rustc_symbol_mangling/src/test.rs
index c6899f8f2..b4d5b7f36 100644
--- a/compiler/rustc_symbol_mangling/src/test.rs
+++ b/compiler/rustc_symbol_mangling/src/test.rs
@@ -53,7 +53,7 @@ impl SymbolNamesTest<'_> {
         // The formatting of `tag({})` is chosen so that tests can elect
         // to test the entirety of the string, if they choose, or else just
         // some subset.
-        for attr in tcx.get_attrs(def_id.to_def_id(), SYMBOL_NAME) {
+        for attr in tcx.get_attrs(def_id, SYMBOL_NAME) {
             let def_id = def_id.to_def_id();
             let instance = Instance::new(
                 def_id,
@@ -79,7 +79,7 @@ impl SymbolNamesTest<'_> {
             }
         }
 
-        for attr in tcx.get_attrs(def_id.to_def_id(), DEF_PATH) {
+        for attr in tcx.get_attrs(def_id, DEF_PATH) {
             tcx.sess.emit_err(TestOutput {
                 span: attr.span,
                 kind: Kind::DefPath,
diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs
index 2f20d4213..ee8832855 100644
--- a/compiler/rustc_symbol_mangling/src/v0.rs
+++ b/compiler/rustc_symbol_mangling/src/v0.rs
@@ -1,5 +1,5 @@
 use rustc_data_structures::base_n;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::intern::Interned;
 use rustc_hir as hir;
 use rustc_hir::def::CtorKind;
@@ -42,6 +42,7 @@ pub(super) fn mangle<'tcx>(
 
     // Append `::{shim:...#0}` to shims that can coexist with a non-shim instance.
     let shim_kind = match instance.def {
+        ty::InstanceDef::ThreadLocalShim(_) => Some("tls"),
         ty::InstanceDef::VTableShim(_) => Some("vtable"),
         ty::InstanceDef::ReifyShim(_) => Some("reify"),
 
@@ -80,9 +81,9 @@ pub(super) fn mangle_typeid_for_trait_ref<'tcx>(
 struct BinderLevel {
     /// The range of distances from the root of what's
     /// being printed, to the lifetimes in a binder.
-    /// Specifically, a `BrAnon(i)` lifetime has depth
-    /// `lifetime_depths.start + i`, going away from the
-    /// the root and towards its use site, as `i` increases.
+    /// Specifically, a `BrAnon` lifetime has depth
+    /// `lifetime_depths.start + index`, going away from the
+    /// the root and towards its use site, as the var index increases.
     /// This is used to flatten rustc's pairing of `BrAnon`
     /// (intra-binder disambiguation) with a `DebruijnIndex`
     /// (binder addressing), to "true" de Bruijn indices,
@@ -207,24 +208,15 @@ impl<'tcx> SymbolMangler<'tcx> {
     where
         T: TypeVisitable<TyCtxt<'tcx>>,
     {
-        // FIXME(non-lifetime-binders): What to do here?
-        let regions = if value.has_late_bound_regions() {
-            self.tcx.collect_referenced_late_bound_regions(value)
-        } else {
-            FxHashSet::default()
-        };
-
         let mut lifetime_depths =
             self.binders.last().map(|b| b.lifetime_depths.end).map_or(0..0, |i| i..i);
 
-        let lifetimes = regions
-            .into_iter()
-            .map(|br| match br {
-                ty::BrAnon(i, _) => i,
-                _ => bug!("symbol_names: non-anonymized region `{:?}` in `{:?}`", br, value),
-            })
-            .max()
-            .map_or(0, |max| max + 1);
+        // FIXME(non-lifetime-binders): What to do here?
+        let lifetimes = value
+            .bound_vars()
+            .iter()
+            .filter(|var| matches!(var, ty::BoundVariableKind::Region(..)))
+            .count() as u32;
 
         self.push_opt_integer_62("G", lifetimes as u64);
         lifetime_depths.end += lifetimes;
@@ -337,9 +329,9 @@ impl<'tcx> Printer<'tcx> for &mut SymbolMangler<'tcx> {
 
             // Late-bound lifetimes use indices starting at 1,
             // see `BinderLevel` for more details.
-            ty::ReLateBound(debruijn, ty::BoundRegion { kind: ty::BrAnon(i, _), .. }) => {
+            ty::ReLateBound(debruijn, ty::BoundRegion { var, kind: ty::BrAnon(_) }) => {
                 let binder = &self.binders[self.binders.len() - 1 - debruijn.index()];
-                let depth = binder.lifetime_depths.start + i;
+                let depth = binder.lifetime_depths.start + var.as_u32();
 
                 1 + (self.binders.last().unwrap().lifetime_depths.end - 1 - depth)
             }
diff --git a/compiler/rustc_target/Cargo.toml b/compiler/rustc_target/Cargo.toml
index 568c916a1..4e7a8d166 100644
--- a/compiler/rustc_target/Cargo.toml
+++ b/compiler/rustc_target/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 bitflags = "1.2.1"
 tracing = "0.1"
 serde_json = "1.0.59"
+rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_abi = { path = "../rustc_abi" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_feature = { path = "../rustc_feature" }
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index a0730fbb6..57011aa8a 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -755,7 +755,7 @@ impl FromStr for Conv {
             "AmdGpuKernel" => Ok(Conv::AmdGpuKernel),
             "AvrInterrupt" => Ok(Conv::AvrInterrupt),
             "AvrNonBlockingInterrupt" => Ok(Conv::AvrNonBlockingInterrupt),
-            _ => Err(format!("'{s}' is not a valid value for entry function call convetion.")),
+            _ => Err(format!("'{s}' is not a valid value for entry function call convention.")),
         }
     }
 }
diff --git a/compiler/rustc_target/src/asm/aarch64.rs b/compiler/rustc_target/src/asm/aarch64.rs
index 28493c770..97132311a 100644
--- a/compiler/rustc_target/src/asm/aarch64.rs
+++ b/compiler/rustc_target/src/asm/aarch64.rs
@@ -1,6 +1,6 @@
 use super::{InlineAsmArch, InlineAsmType};
 use crate::spec::{RelocModel, Target};
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_macros::HashStable_Generic;
 use rustc_span::Symbol;
 use std::fmt;
@@ -80,7 +80,7 @@ pub fn target_reserves_x18(target: &Target) -> bool {
 fn reserved_x18(
     _arch: InlineAsmArch,
     _reloc_model: RelocModel,
-    _target_features: &FxHashSet<Symbol>,
+    _target_features: &FxIndexSet<Symbol>,
     target: &Target,
     _is_clobber: bool,
 ) -> Result<(), &'static str> {
diff --git a/compiler/rustc_target/src/asm/arm.rs b/compiler/rustc_target/src/asm/arm.rs
index ec7429a30..514e30ae0 100644
--- a/compiler/rustc_target/src/asm/arm.rs
+++ b/compiler/rustc_target/src/asm/arm.rs
@@ -1,6 +1,6 @@
 use super::{InlineAsmArch, InlineAsmType};
 use crate::spec::{RelocModel, Target};
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_macros::HashStable_Generic;
 use rustc_span::{sym, Symbol};
 use std::fmt;
@@ -64,14 +64,14 @@ impl ArmInlineAsmRegClass {
 }
 
 // This uses the same logic as useR7AsFramePointer in LLVM
-fn frame_pointer_is_r7(target_features: &FxHashSet<Symbol>, target: &Target) -> bool {
+fn frame_pointer_is_r7(target_features: &FxIndexSet<Symbol>, target: &Target) -> bool {
     target.is_like_osx || (!target.is_like_windows && target_features.contains(&sym::thumb_mode))
 }
 
 fn frame_pointer_r11(
     arch: InlineAsmArch,
     reloc_model: RelocModel,
-    target_features: &FxHashSet<Symbol>,
+    target_features: &FxIndexSet<Symbol>,
     target: &Target,
     is_clobber: bool,
 ) -> Result<(), &'static str> {
@@ -87,7 +87,7 @@ fn frame_pointer_r11(
 fn frame_pointer_r7(
     _arch: InlineAsmArch,
     _reloc_model: RelocModel,
-    target_features: &FxHashSet<Symbol>,
+    target_features: &FxIndexSet<Symbol>,
     target: &Target,
     _is_clobber: bool,
 ) -> Result<(), &'static str> {
@@ -101,7 +101,7 @@ fn frame_pointer_r7(
 fn not_thumb1(
     _arch: InlineAsmArch,
     _reloc_model: RelocModel,
-    target_features: &FxHashSet<Symbol>,
+    target_features: &FxIndexSet<Symbol>,
     _target: &Target,
     is_clobber: bool,
 ) -> Result<(), &'static str> {
@@ -118,7 +118,7 @@ fn not_thumb1(
 fn reserved_r9(
     arch: InlineAsmArch,
     reloc_model: RelocModel,
-    target_features: &FxHashSet<Symbol>,
+    target_features: &FxIndexSet<Symbol>,
     target: &Target,
     is_clobber: bool,
 ) -> Result<(), &'static str> {
diff --git a/compiler/rustc_target/src/asm/m68k.rs b/compiler/rustc_target/src/asm/m68k.rs
new file mode 100644
index 000000000..8c857550c
--- /dev/null
+++ b/compiler/rustc_target/src/asm/m68k.rs
@@ -0,0 +1,81 @@
+use super::{InlineAsmArch, InlineAsmType};
+use rustc_macros::HashStable_Generic;
+use rustc_span::Symbol;
+use std::fmt;
+
+def_reg_class! {
+    M68k M68kInlineAsmRegClass {
+        reg,
+        reg_addr,
+        reg_data,
+    }
+}
+
+impl M68kInlineAsmRegClass {
+    pub fn valid_modifiers(self, _arch: super::InlineAsmArch) -> &'static [char] {
+        &[]
+    }
+
+    pub fn suggest_class(self, _arch: InlineAsmArch, _ty: InlineAsmType) -> Option<Self> {
+        None
+    }
+
+    pub fn suggest_modifier(
+        self,
+        _arch: InlineAsmArch,
+        _ty: InlineAsmType,
+    ) -> Option<(char, &'static str)> {
+        None
+    }
+
+    pub fn default_modifier(self, _arch: InlineAsmArch) -> Option<(char, &'static str)> {
+        None
+    }
+
+    pub fn supported_types(
+        self,
+        _arch: InlineAsmArch,
+    ) -> &'static [(InlineAsmType, Option<Symbol>)] {
+        match self {
+            Self::reg => types! { _: I16, I32; },
+            Self::reg_data => types! { _: I8, I16, I32; },
+            Self::reg_addr => types! { _: I16, I32; },
+        }
+    }
+}
+
+def_regs! {
+    M68k M68kInlineAsmReg M68kInlineAsmRegClass {
+        d0: reg, reg_data = ["d0"],
+        d1: reg, reg_data = ["d1"],
+        d2: reg, reg_data = ["d2"],
+        d3: reg, reg_data = ["d3"],
+        d4: reg, reg_data = ["d4"],
+        d5: reg, reg_data = ["d5"],
+        d6: reg, reg_data = ["d6"],
+        d7: reg, reg_data = ["d7"],
+        a0: reg, reg_addr = ["a0"],
+        a1: reg, reg_addr = ["a1"],
+        a2: reg, reg_addr = ["a2"],
+        a3: reg, reg_addr = ["a3"],
+        #error = ["a4"] =>
+            "a4 is used internally by LLVM and cannot be used as an operand for inline asm",
+        #error = ["a5", "bp"] =>
+            "a5 is used internally by LLVM and cannot be used as an operand for inline asm",
+        #error = ["a6", "fp"] =>
+            "a6 is used internally by LLVM and cannot be used as an operand for inline asm",
+        #error = ["a7", "sp", "usp", "ssp", "isp"] =>
+            "the stack pointer cannot be used as an operand for inline asm",
+    }
+}
+
+impl M68kInlineAsmReg {
+    pub fn emit(
+        self,
+        out: &mut dyn fmt::Write,
+        _arch: InlineAsmArch,
+        _modifier: Option<char>,
+    ) -> fmt::Result {
+        out.write_str(self.name())
+    }
+}
diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs
index 70cd883be..3f9c850b3 100644
--- a/compiler/rustc_target/src/asm/mod.rs
+++ b/compiler/rustc_target/src/asm/mod.rs
@@ -1,6 +1,6 @@
 use crate::spec::Target;
 use crate::{abi::Size, spec::RelocModel};
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_macros::HashStable_Generic;
 use rustc_span::Symbol;
 use std::fmt;
@@ -37,13 +37,14 @@ macro_rules! def_reg_class {
 
         pub(super) fn regclass_map() -> rustc_data_structures::fx::FxHashMap<
             super::InlineAsmRegClass,
-            rustc_data_structures::fx::FxHashSet<super::InlineAsmReg>,
+            rustc_data_structures::fx::FxIndexSet<super::InlineAsmReg>,
         > {
-            use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+            use rustc_data_structures::fx::FxHashMap;
+            use rustc_data_structures::fx::FxIndexSet;
             use super::InlineAsmRegClass;
             let mut map = FxHashMap::default();
             $(
-                map.insert(InlineAsmRegClass::$arch($arch_regclass::$class), FxHashSet::default());
+                map.insert(InlineAsmRegClass::$arch($arch_regclass::$class), FxIndexSet::default());
             )*
             map
         }
@@ -94,7 +95,7 @@ macro_rules! def_regs {
             pub fn validate(self,
                 _arch: super::InlineAsmArch,
                 _reloc_model: crate::spec::RelocModel,
-                _target_features: &rustc_data_structures::fx::FxHashSet<Symbol>,
+                _target_features: &rustc_data_structures::fx::FxIndexSet<Symbol>,
                 _target: &crate::spec::Target,
                 _is_clobber: bool,
             ) -> Result<(), &'static str> {
@@ -118,11 +119,11 @@ macro_rules! def_regs {
         pub(super) fn fill_reg_map(
             _arch: super::InlineAsmArch,
             _reloc_model: crate::spec::RelocModel,
-            _target_features: &rustc_data_structures::fx::FxHashSet<Symbol>,
+            _target_features: &rustc_data_structures::fx::FxIndexSet<Symbol>,
             _target: &crate::spec::Target,
             _map: &mut rustc_data_structures::fx::FxHashMap<
                 super::InlineAsmRegClass,
-                rustc_data_structures::fx::FxHashSet<super::InlineAsmReg>,
+                rustc_data_structures::fx::FxIndexSet<super::InlineAsmReg>,
             >,
         ) {
             #[allow(unused_imports)]
@@ -167,6 +168,7 @@ mod arm;
 mod avr;
 mod bpf;
 mod hexagon;
+mod m68k;
 mod mips;
 mod msp430;
 mod nvptx;
@@ -182,6 +184,7 @@ pub use arm::{ArmInlineAsmReg, ArmInlineAsmRegClass};
 pub use avr::{AvrInlineAsmReg, AvrInlineAsmRegClass};
 pub use bpf::{BpfInlineAsmReg, BpfInlineAsmRegClass};
 pub use hexagon::{HexagonInlineAsmReg, HexagonInlineAsmRegClass};
+pub use m68k::{M68kInlineAsmReg, M68kInlineAsmRegClass};
 pub use mips::{MipsInlineAsmReg, MipsInlineAsmRegClass};
 pub use msp430::{Msp430InlineAsmReg, Msp430InlineAsmRegClass};
 pub use nvptx::{NvptxInlineAsmReg, NvptxInlineAsmRegClass};
@@ -213,6 +216,7 @@ pub enum InlineAsmArch {
     Bpf,
     Avr,
     Msp430,
+    M68k,
 }
 
 impl FromStr for InlineAsmArch {
@@ -239,6 +243,7 @@ impl FromStr for InlineAsmArch {
             "bpf" => Ok(Self::Bpf),
             "avr" => Ok(Self::Avr),
             "msp430" => Ok(Self::Msp430),
+            "m68k" => Ok(Self::M68k),
             _ => Err(()),
         }
     }
@@ -261,6 +266,7 @@ pub enum InlineAsmReg {
     Bpf(BpfInlineAsmReg),
     Avr(AvrInlineAsmReg),
     Msp430(Msp430InlineAsmReg),
+    M68k(M68kInlineAsmReg),
     // Placeholder for invalid register constraints for the current target
     Err,
 }
@@ -279,6 +285,7 @@ impl InlineAsmReg {
             Self::Bpf(r) => r.name(),
             Self::Avr(r) => r.name(),
             Self::Msp430(r) => r.name(),
+            Self::M68k(r) => r.name(),
             Self::Err => "<reg>",
         }
     }
@@ -296,6 +303,7 @@ impl InlineAsmReg {
             Self::Bpf(r) => InlineAsmRegClass::Bpf(r.reg_class()),
             Self::Avr(r) => InlineAsmRegClass::Avr(r.reg_class()),
             Self::Msp430(r) => InlineAsmRegClass::Msp430(r.reg_class()),
+            Self::M68k(r) => InlineAsmRegClass::M68k(r.reg_class()),
             Self::Err => InlineAsmRegClass::Err,
         }
     }
@@ -327,6 +335,7 @@ impl InlineAsmReg {
             InlineAsmArch::Bpf => Self::Bpf(BpfInlineAsmReg::parse(name)?),
             InlineAsmArch::Avr => Self::Avr(AvrInlineAsmReg::parse(name)?),
             InlineAsmArch::Msp430 => Self::Msp430(Msp430InlineAsmReg::parse(name)?),
+            InlineAsmArch::M68k => Self::M68k(M68kInlineAsmReg::parse(name)?),
         })
     }
 
@@ -334,7 +343,7 @@ impl InlineAsmReg {
         self,
         arch: InlineAsmArch,
         reloc_model: RelocModel,
-        target_features: &FxHashSet<Symbol>,
+        target_features: &FxIndexSet<Symbol>,
         target: &Target,
         is_clobber: bool,
     ) -> Result<(), &'static str> {
@@ -350,6 +359,7 @@ impl InlineAsmReg {
             Self::Bpf(r) => r.validate(arch, reloc_model, target_features, target, is_clobber),
             Self::Avr(r) => r.validate(arch, reloc_model, target_features, target, is_clobber),
             Self::Msp430(r) => r.validate(arch, reloc_model, target_features, target, is_clobber),
+            Self::M68k(r) => r.validate(arch, reloc_model, target_features, target, is_clobber),
             Self::Err => unreachable!(),
         }
     }
@@ -374,6 +384,7 @@ impl InlineAsmReg {
             Self::Bpf(r) => r.emit(out, arch, modifier),
             Self::Avr(r) => r.emit(out, arch, modifier),
             Self::Msp430(r) => r.emit(out, arch, modifier),
+            Self::M68k(r) => r.emit(out, arch, modifier),
             Self::Err => unreachable!("Use of InlineAsmReg::Err"),
         }
     }
@@ -391,6 +402,7 @@ impl InlineAsmReg {
             Self::Bpf(r) => r.overlapping_regs(|r| cb(Self::Bpf(r))),
             Self::Avr(r) => r.overlapping_regs(|r| cb(Self::Avr(r))),
             Self::Msp430(_) => cb(self),
+            Self::M68k(_) => cb(self),
             Self::Err => unreachable!("Use of InlineAsmReg::Err"),
         }
     }
@@ -413,6 +425,7 @@ pub enum InlineAsmRegClass {
     Bpf(BpfInlineAsmRegClass),
     Avr(AvrInlineAsmRegClass),
     Msp430(Msp430InlineAsmRegClass),
+    M68k(M68kInlineAsmRegClass),
     // Placeholder for invalid register constraints for the current target
     Err,
 }
@@ -434,6 +447,7 @@ impl InlineAsmRegClass {
             Self::Bpf(r) => r.name(),
             Self::Avr(r) => r.name(),
             Self::Msp430(r) => r.name(),
+            Self::M68k(r) => r.name(),
             Self::Err => rustc_span::symbol::sym::reg,
         }
     }
@@ -457,6 +471,7 @@ impl InlineAsmRegClass {
             Self::Bpf(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Bpf),
             Self::Avr(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Avr),
             Self::Msp430(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Msp430),
+            Self::M68k(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::M68k),
             Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
@@ -487,6 +502,7 @@ impl InlineAsmRegClass {
             Self::Bpf(r) => r.suggest_modifier(arch, ty),
             Self::Avr(r) => r.suggest_modifier(arch, ty),
             Self::Msp430(r) => r.suggest_modifier(arch, ty),
+            Self::M68k(r) => r.suggest_modifier(arch, ty),
             Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
@@ -513,6 +529,7 @@ impl InlineAsmRegClass {
             Self::Bpf(r) => r.default_modifier(arch),
             Self::Avr(r) => r.default_modifier(arch),
             Self::Msp430(r) => r.default_modifier(arch),
+            Self::M68k(r) => r.default_modifier(arch),
             Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
@@ -538,6 +555,7 @@ impl InlineAsmRegClass {
             Self::Bpf(r) => r.supported_types(arch),
             Self::Avr(r) => r.supported_types(arch),
             Self::Msp430(r) => r.supported_types(arch),
+            Self::M68k(r) => r.supported_types(arch),
             Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
@@ -568,6 +586,7 @@ impl InlineAsmRegClass {
             InlineAsmArch::Bpf => Self::Bpf(BpfInlineAsmRegClass::parse(name)?),
             InlineAsmArch::Avr => Self::Avr(AvrInlineAsmRegClass::parse(name)?),
             InlineAsmArch::Msp430 => Self::Msp430(Msp430InlineAsmRegClass::parse(name)?),
+            InlineAsmArch::M68k => Self::M68k(M68kInlineAsmRegClass::parse(name)?),
         })
     }
 
@@ -589,6 +608,7 @@ impl InlineAsmRegClass {
             Self::Bpf(r) => r.valid_modifiers(arch),
             Self::Avr(r) => r.valid_modifiers(arch),
             Self::Msp430(r) => r.valid_modifiers(arch),
+            Self::M68k(r) => r.valid_modifiers(arch),
             Self::Err => unreachable!("Use of InlineAsmRegClass::Err"),
         }
     }
@@ -701,9 +721,9 @@ impl fmt::Display for InlineAsmType {
 pub fn allocatable_registers(
     arch: InlineAsmArch,
     reloc_model: RelocModel,
-    target_features: &FxHashSet<Symbol>,
+    target_features: &FxIndexSet<Symbol>,
     target: &crate::spec::Target,
-) -> FxHashMap<InlineAsmRegClass, FxHashSet<InlineAsmReg>> {
+) -> FxHashMap<InlineAsmRegClass, FxIndexSet<InlineAsmReg>> {
     match arch {
         InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
             let mut map = x86::regclass_map();
@@ -775,6 +795,11 @@ pub fn allocatable_registers(
             msp430::fill_reg_map(arch, reloc_model, target_features, target, &mut map);
             map
         }
+        InlineAsmArch::M68k => {
+            let mut map = m68k::regclass_map();
+            m68k::fill_reg_map(arch, reloc_model, target_features, target, &mut map);
+            map
+        }
     }
 }
 
diff --git a/compiler/rustc_target/src/asm/riscv.rs b/compiler/rustc_target/src/asm/riscv.rs
index e41bdc9a5..dea6d50fe 100644
--- a/compiler/rustc_target/src/asm/riscv.rs
+++ b/compiler/rustc_target/src/asm/riscv.rs
@@ -1,6 +1,6 @@
 use super::{InlineAsmArch, InlineAsmType};
 use crate::spec::{RelocModel, Target};
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_macros::HashStable_Generic;
 use rustc_span::{sym, Symbol};
 use std::fmt;
@@ -55,7 +55,7 @@ impl RiscVInlineAsmRegClass {
 fn not_e(
     _arch: InlineAsmArch,
     _reloc_model: RelocModel,
-    target_features: &FxHashSet<Symbol>,
+    target_features: &FxIndexSet<Symbol>,
     _target: &Target,
     _is_clobber: bool,
 ) -> Result<(), &'static str> {
diff --git a/compiler/rustc_target/src/asm/x86.rs b/compiler/rustc_target/src/asm/x86.rs
index 5eae07f14..3902dac7f 100644
--- a/compiler/rustc_target/src/asm/x86.rs
+++ b/compiler/rustc_target/src/asm/x86.rs
@@ -1,6 +1,6 @@
 use super::{InlineAsmArch, InlineAsmType};
 use crate::spec::{RelocModel, Target};
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_macros::HashStable_Generic;
 use rustc_span::Symbol;
 use std::fmt;
@@ -147,7 +147,7 @@ impl X86InlineAsmRegClass {
 fn x86_64_only(
     arch: InlineAsmArch,
     _reloc_model: RelocModel,
-    _target_features: &FxHashSet<Symbol>,
+    _target_features: &FxIndexSet<Symbol>,
     _target: &Target,
     _is_clobber: bool,
 ) -> Result<(), &'static str> {
@@ -161,7 +161,7 @@ fn x86_64_only(
 fn high_byte(
     arch: InlineAsmArch,
     _reloc_model: RelocModel,
-    _target_features: &FxHashSet<Symbol>,
+    _target_features: &FxIndexSet<Symbol>,
     _target: &Target,
     _is_clobber: bool,
 ) -> Result<(), &'static str> {
@@ -174,7 +174,7 @@ fn high_byte(
 fn rbx_reserved(
     arch: InlineAsmArch,
     _reloc_model: RelocModel,
-    _target_features: &FxHashSet<Symbol>,
+    _target_features: &FxIndexSet<Symbol>,
     _target: &Target,
     _is_clobber: bool,
 ) -> Result<(), &'static str> {
@@ -190,7 +190,7 @@ fn rbx_reserved(
 fn esi_reserved(
     arch: InlineAsmArch,
     _reloc_model: RelocModel,
-    _target_features: &FxHashSet<Symbol>,
+    _target_features: &FxIndexSet<Symbol>,
     _target: &Target,
     _is_clobber: bool,
 ) -> Result<(), &'static str> {
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_linux_ohos.rs b/compiler/rustc_target/src/spec/aarch64_unknown_linux_ohos.rs
new file mode 100644
index 000000000..bf1b089f6
--- /dev/null
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_linux_ohos.rs
@@ -0,0 +1,32 @@
+use crate::spec::{Target, TargetOptions};
+
+use super::SanitizerSet;
+
+pub fn target() -> Target {
+    let mut base = super::linux_musl_base::opts();
+    base.env = "ohos".into();
+    base.crt_static_default = false;
+    base.max_atomic_width = Some(128);
+
+    Target {
+        // LLVM 15 doesn't support OpenHarmony yet, use a linux target instead.
+        llvm_target: "aarch64-unknown-linux-musl".into(),
+        pointer_width: 64,
+        data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
+        arch: "aarch64".into(),
+        options: TargetOptions {
+            features: "+reserve-x18".into(),
+            mcount: "\u{1}_mcount".into(),
+            force_emulated_tls: true,
+            has_thread_local: false,
+            supported_sanitizers: SanitizerSet::ADDRESS
+                | SanitizerSet::CFI
+                | SanitizerSet::LEAK
+                | SanitizerSet::MEMORY
+                | SanitizerSet::MEMTAG
+                | SanitizerSet::THREAD
+                | SanitizerSet::HWADDRESS,
+            ..base
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_nto_qnx_710.rs b/compiler/rustc_target/src/spec/aarch64_unknown_nto_qnx_710.rs
index 8c1126ae6..630642dcd 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_nto_qnx_710.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_nto_qnx_710.rs
@@ -23,6 +23,7 @@ pub fn target() -> Target {
                 LinkerFlavor::Gnu(Cc::Yes, Lld::No),
                 &["-Vgcc_ntoaarch64le_cxx"],
             ),
+            env: "nto71".into(),
             ..nto_qnx_base::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/abi.rs b/compiler/rustc_target/src/spec/abi.rs
index d4f7ed31b..5582d909f 100644
--- a/compiler/rustc_target/src/spec/abi.rs
+++ b/compiler/rustc_target/src/spec/abi.rs
@@ -324,8 +324,6 @@ impl Abi {
 
 impl fmt::Display for Abi {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            abi => write!(f, "\"{}\"", abi.name()),
-        }
+        write!(f, "\"{}\"", self.name())
     }
 }
diff --git a/compiler/rustc_target/src/spec/armv4t_none_eabi.rs b/compiler/rustc_target/src/spec/armv4t_none_eabi.rs
index 28b109889..d0f988b27 100644
--- a/compiler/rustc_target/src/spec/armv4t_none_eabi.rs
+++ b/compiler/rustc_target/src/spec/armv4t_none_eabi.rs
@@ -4,13 +4,6 @@
 //!
 //! Please ping @Lokathor if changes are needed.
 //!
-//! This target profile assumes that you have the ARM binutils in your path
-//! (specifically the linker, `arm-none-eabi-ld`). They can be obtained for free
-//! for all major OSes from the ARM developer's website, and they may also be
-//! available in your system's package manager. Unfortunately, the standard
-//! linker that Rust uses (`lld`) only supports as far back as `ARMv5TE`, so we
-//! must use the GNU `ld` linker.
-//!
 //! **Important:** This target profile **does not** specify a linker script. You
 //! just get the default link script when you build a binary for this target.
 //! The default link script is very likely wrong, so you should use
@@ -35,8 +28,8 @@ pub fn target() -> Target {
         data_layout: "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64".into(),
         options: TargetOptions {
             abi: "eabi".into(),
-            linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::No),
-            linker: Some("arm-none-eabi-ld".into()),
+            linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
+            linker: Some("rust-lld".into()),
             asm_args: cvs!["-mthumb-interwork", "-march=armv4t", "-mlittle-endian",],
             // Force-enable 32-bit atomics, which allows the use of atomic load/store only.
             // The resulting atomics are ABI incompatible with atomics backed by libatomic.
diff --git a/compiler/rustc_target/src/spec/armv7_unknown_linux_ohos.rs b/compiler/rustc_target/src/spec/armv7_unknown_linux_ohos.rs
new file mode 100644
index 000000000..16da24533
--- /dev/null
+++ b/compiler/rustc_target/src/spec/armv7_unknown_linux_ohos.rs
@@ -0,0 +1,28 @@
+use crate::spec::{Target, TargetOptions};
+
+// This target is for OpenHarmony on ARMv7 Linux with thumb-mode, but no NEON or
+// hardfloat.
+
+pub fn target() -> Target {
+    // Most of these settings are copied from the armv7_unknown_linux_musleabi
+    // target.
+    Target {
+        // LLVM 15 doesn't support OpenHarmony yet, use a linux target instead.
+        llvm_target: "armv7-unknown-linux-gnueabi".into(),
+        pointer_width: 32,
+        data_layout: "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64".into(),
+        arch: "arm".into(),
+
+        options: TargetOptions {
+            abi: "eabi".into(),
+            features: "+v7,+thumb2,+soft-float,-neon".into(),
+            max_atomic_width: Some(64),
+            env: "ohos".into(),
+            crt_static_default: false,
+            mcount: "\u{1}mcount".into(),
+            force_emulated_tls: true,
+            has_thread_local: false,
+            ..super::linux_musl_base::opts()
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/i586_pc_nto_qnx700.rs b/compiler/rustc_target/src/spec/i586_pc_nto_qnx700.rs
new file mode 100644
index 000000000..68afa7fe4
--- /dev/null
+++ b/compiler/rustc_target/src/spec/i586_pc_nto_qnx700.rs
@@ -0,0 +1,24 @@
+use super::nto_qnx_base;
+use crate::spec::{Cc, LinkerFlavor, Lld, StackProbeType, Target, TargetOptions};
+
+pub fn target() -> Target {
+    Target {
+        llvm_target: "i586-pc-unknown".into(),
+        pointer_width: 32,
+        data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
+            f64:32:64-f80:32-n8:16:32-S128"
+            .into(),
+        arch: "x86".into(),
+        options: TargetOptions {
+            cpu: "pentium4".into(),
+            max_atomic_width: Some(64),
+            pre_link_args: TargetOptions::link_args(
+                LinkerFlavor::Gnu(Cc::Yes, Lld::No),
+                &["-Vgcc_ntox86_cxx"],
+            ),
+            env: "nto70".into(),
+            stack_probes: StackProbeType::X86,
+            ..nto_qnx_base::opts()
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/loongarch64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/loongarch64_unknown_linux_gnu.rs
new file mode 100644
index 000000000..db8b9c70e
--- /dev/null
+++ b/compiler/rustc_target/src/spec/loongarch64_unknown_linux_gnu.rs
@@ -0,0 +1,17 @@
+use crate::spec::{Target, TargetOptions};
+
+pub fn target() -> Target {
+    Target {
+        llvm_target: "loongarch64-unknown-linux-gnu".into(),
+        pointer_width: 64,
+        data_layout: "e-m:e-p:64:64-i64:64-i128:128-n64-S128".into(),
+        arch: "loongarch64".into(),
+        options: TargetOptions {
+            cpu: "generic".into(),
+            features: "+f,+d".into(),
+            llvm_abiname: "lp64d".into(),
+            max_atomic_width: Some(64),
+            ..super::linux_gnu_base::opts()
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 0d86a3032..4e5a821f0 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -40,6 +40,7 @@ use crate::json::{Json, ToJson};
 use crate::spec::abi::{lookup as lookup_abi, Abi};
 use crate::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_fs_util::try_canonicalize;
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use rustc_span::symbol::{sym, Symbol};
 use serde_json::Value;
@@ -122,7 +123,7 @@ pub enum Lld {
 /// target properties, in accordance with the first design goal.
 ///
 /// The first component of the flavor is tightly coupled with the compilation target,
-/// while the `Cc` and `Lld` flags can vary withing the same target.
+/// while the `Cc` and `Lld` flags can vary within the same target.
 #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
 pub enum LinkerFlavor {
     /// Unix-like linker with GNU extensions (both naked and compiler-wrapped forms).
@@ -1020,6 +1021,7 @@ supported_targets! {
     ("x86_64-unknown-linux-gnux32", x86_64_unknown_linux_gnux32),
     ("i686-unknown-linux-gnu", i686_unknown_linux_gnu),
     ("i586-unknown-linux-gnu", i586_unknown_linux_gnu),
+    ("loongarch64-unknown-linux-gnu", loongarch64_unknown_linux_gnu),
     ("m68k-unknown-linux-gnu", m68k_unknown_linux_gnu),
     ("mips-unknown-linux-gnu", mips_unknown_linux_gnu),
     ("mips64-unknown-linux-gnuabi64", mips64_unknown_linux_gnuabi64),
@@ -1115,6 +1117,7 @@ supported_targets! {
     // FIXME(#106649): Remove aarch64-fuchsia in favor of aarch64-unknown-fuchsia
     ("aarch64-fuchsia", aarch64_fuchsia),
     ("aarch64-unknown-fuchsia", aarch64_unknown_fuchsia),
+    ("riscv64gc-unknown-fuchsia", riscv64gc_unknown_fuchsia),
     // FIXME(#106649): Remove x86_64-fuchsia in favor of x86_64-unknown-fuchsia
     ("x86_64-fuchsia", x86_64_fuchsia),
     ("x86_64-unknown-fuchsia", x86_64_unknown_fuchsia),
@@ -1259,6 +1262,10 @@ supported_targets! {
 
     ("aarch64-unknown-nto-qnx710", aarch64_unknown_nto_qnx_710),
     ("x86_64-pc-nto-qnx710", x86_64_pc_nto_qnx710),
+    ("i586-pc-nto-qnx700", i586_pc_nto_qnx700),
+
+    ("aarch64-unknown-linux-ohos", aarch64_unknown_linux_ohos),
+    ("armv7-unknown-linux-ohos", armv7_unknown_linux_ohos),
 }
 
 /// Cow-Vec-Str: Cow<'static, [Cow<'static, str>]>
@@ -1466,6 +1473,8 @@ pub struct TargetOptions {
     pub features: StaticCow<str>,
     /// Whether dynamic linking is available on this target. Defaults to false.
     pub dynamic_linking: bool,
+    /// Whether dynamic linking can export TLS globals. Defaults to true.
+    pub dll_tls_export: bool,
     /// If dynamic linking is available, whether only cdylibs are supported.
     pub only_cdylib: bool,
     /// Whether executables are available on this target. Defaults to true.
@@ -1732,6 +1741,9 @@ pub struct TargetOptions {
 
     /// Whether the target supports XRay instrumentation.
     pub supports_xray: bool,
+
+    /// Forces the use of emulated TLS (__emutls_get_address)
+    pub force_emulated_tls: bool,
 }
 
 /// Add arguments for the given flavor and also for its "twin" flavors
@@ -1857,6 +1869,7 @@ impl Default for TargetOptions {
             cpu: "generic".into(),
             features: "".into(),
             dynamic_linking: false,
+            dll_tls_export: true,
             only_cdylib: false,
             executables: true,
             relocation_model: RelocModel::Pic,
@@ -1952,6 +1965,7 @@ impl Default for TargetOptions {
             entry_name: "main".into(),
             entry_abi: Conv::C,
             supports_xray: false,
+            force_emulated_tls: false,
         }
     }
 }
@@ -2528,6 +2542,7 @@ impl Target {
         key!(cpu);
         key!(features);
         key!(dynamic_linking, bool);
+        key!(dll_tls_export, bool);
         key!(only_cdylib, bool);
         key!(executables, bool);
         key!(relocation_model, RelocModel)?;
@@ -2603,6 +2618,7 @@ impl Target {
         key!(entry_name);
         key!(entry_abi, Conv)?;
         key!(supports_xray, bool);
+        key!(force_emulated_tls, bool);
 
         if base.is_builtin {
             // This can cause unfortunate ICEs later down the line.
@@ -2781,6 +2797,7 @@ impl ToJson for Target {
         target_option_val!(cpu);
         target_option_val!(features);
         target_option_val!(dynamic_linking);
+        target_option_val!(dll_tls_export);
         target_option_val!(only_cdylib);
         target_option_val!(executables);
         target_option_val!(relocation_model);
@@ -2857,6 +2874,7 @@ impl ToJson for Target {
         target_option_val!(entry_name);
         target_option_val!(entry_abi);
         target_option_val!(supports_xray);
+        target_option_val!(force_emulated_tls);
 
         if let Some(abi) = self.default_adjusted_cabi {
             d.insert("default-adjusted-cabi".into(), Abi::name(abi).to_json());
@@ -2948,7 +2966,7 @@ impl TargetTriple {
 
     /// Creates a target triple from the passed target path.
     pub fn from_path(path: &Path) -> Result<Self, io::Error> {
-        let canonicalized_path = path.canonicalize()?;
+        let canonicalized_path = try_canonicalize(path)?;
         let contents = std::fs::read_to_string(&canonicalized_path).map_err(|err| {
             io::Error::new(
                 io::ErrorKind::InvalidInput,
diff --git a/compiler/rustc_target/src/spec/msvc_base.rs b/compiler/rustc_target/src/spec/msvc_base.rs
index 1dad9133e..efe949a4e 100644
--- a/compiler/rustc_target/src/spec/msvc_base.rs
+++ b/compiler/rustc_target/src/spec/msvc_base.rs
@@ -8,6 +8,7 @@ pub fn opts() -> TargetOptions {
 
     TargetOptions {
         linker_flavor: LinkerFlavor::Msvc(Lld::No),
+        dll_tls_export: false,
         is_like_windows: true,
         is_like_msvc: true,
         pre_link_args,
diff --git a/compiler/rustc_target/src/spec/nto_qnx_base.rs b/compiler/rustc_target/src/spec/nto_qnx_base.rs
index 6fb581ef5..f1405e9b4 100644
--- a/compiler/rustc_target/src/spec/nto_qnx_base.rs
+++ b/compiler/rustc_target/src/spec/nto_qnx_base.rs
@@ -4,7 +4,6 @@ pub fn opts() -> TargetOptions {
     TargetOptions {
         crt_static_respected: true,
         dynamic_linking: true,
-        env: "nto71".into(),
         executables: true,
         families: cvs!["unix"],
         has_rpath: true,
diff --git a/compiler/rustc_target/src/spec/riscv64gc_unknown_fuchsia.rs b/compiler/rustc_target/src/spec/riscv64gc_unknown_fuchsia.rs
new file mode 100644
index 000000000..0585ed76f
--- /dev/null
+++ b/compiler/rustc_target/src/spec/riscv64gc_unknown_fuchsia.rs
@@ -0,0 +1,19 @@
+use crate::spec::{CodeModel, SanitizerSet, Target, TargetOptions};
+
+pub fn target() -> Target {
+    Target {
+        llvm_target: "riscv64-unknown-fuchsia".into(),
+        pointer_width: 64,
+        data_layout: "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128".into(),
+        arch: "riscv64".into(),
+        options: TargetOptions {
+            code_model: Some(CodeModel::Medium),
+            cpu: "generic-rv64".into(),
+            features: "+m,+a,+f,+d,+c".into(),
+            llvm_abiname: "lp64d".into(),
+            max_atomic_width: Some(64),
+            supported_sanitizers: SanitizerSet::SHADOWCALLSTACK,
+            ..super::fuchsia_base::opts()
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs b/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs
index e3734932f..9c59bb911 100644
--- a/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs
+++ b/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs
@@ -4,19 +4,12 @@
 //!
 //! Please ping @Lokathor if changes are needed.
 //!
-//! This target profile assumes that you have the ARM binutils in your path
-//! (specifically the linker, `arm-none-eabi-ld`). They can be obtained for free
-//! for all major OSes from the ARM developer's website, and they may also be
-//! available in your system's package manager. Unfortunately, the standard
-//! linker that Rust uses (`lld`) only supports as far back as `ARMv5TE`, so we
-//! must use the GNU `ld` linker.
-//!
 //! **Important:** This target profile **does not** specify a linker script. You
 //! just get the default link script when you build a binary for this target.
 //! The default link script is very likely wrong, so you should use
 //! `-Clink-arg=-Tmy_script.ld` to override that with a correct linker script.
 
-use crate::spec::{cvs, Cc, FramePointer, LinkerFlavor, Lld};
+use crate::spec::{cvs, FramePointer};
 use crate::spec::{PanicStrategy, RelocModel, Target, TargetOptions};
 
 pub fn target() -> Target {
@@ -36,8 +29,6 @@ pub fn target() -> Target {
         data_layout: "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64".into(),
         options: TargetOptions {
             abi: "eabi".into(),
-            linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::No),
-            linker: Some("arm-none-eabi-ld".into()),
 
             // extra args passed to the external assembler (assuming `arm-none-eabi-as`):
             // * activate t32/a32 interworking
diff --git a/compiler/rustc_target/src/spec/wasm_base.rs b/compiler/rustc_target/src/spec/wasm_base.rs
index 625d3b37c..341763aad 100644
--- a/compiler/rustc_target/src/spec/wasm_base.rs
+++ b/compiler/rustc_target/src/spec/wasm_base.rs
@@ -1,3 +1,4 @@
+use super::crt_objects::LinkSelfContainedDefault;
 use super::{cvs, Cc, LinkerFlavor, PanicStrategy, RelocModel, TargetOptions, TlsModel};
 
 pub fn options() -> TargetOptions {
@@ -94,6 +95,13 @@ pub fn options() -> TargetOptions {
 
         pre_link_args,
 
+        // FIXME: Figure out cases in which WASM needs to link with a native toolchain.
+        //
+        // rust-lang/rust#104137: cannot blindly remove this without putting in
+        // some other way to compensate for lack of `-nostartfiles` in linker
+        // invocation.
+        link_self_contained: LinkSelfContainedDefault::True,
+
         // This has no effect in LLVM 8 or prior, but in LLVM 9 and later when
         // PIC code is implemented this has quite a drastic effect if it stays
         // at the default, `pic`. In an effort to keep wasm binaries as minimal
diff --git a/compiler/rustc_target/src/spec/windows_gnu_base.rs b/compiler/rustc_target/src/spec/windows_gnu_base.rs
index a32ca469b..2231983f0 100644
--- a/compiler/rustc_target/src/spec/windows_gnu_base.rs
+++ b/compiler/rustc_target/src/spec/windows_gnu_base.rs
@@ -78,6 +78,7 @@ pub fn opts() -> TargetOptions {
         function_sections: false,
         linker: Some("gcc".into()),
         dynamic_linking: true,
+        dll_tls_export: false,
         dll_prefix: "".into(),
         dll_suffix: ".dll".into(),
         exe_suffix: ".exe".into(),
diff --git a/compiler/rustc_target/src/spec/windows_gnullvm_base.rs b/compiler/rustc_target/src/spec/windows_gnullvm_base.rs
index cada28652..b1d8e2be5 100644
--- a/compiler/rustc_target/src/spec/windows_gnullvm_base.rs
+++ b/compiler/rustc_target/src/spec/windows_gnullvm_base.rs
@@ -23,6 +23,7 @@ pub fn opts() -> TargetOptions {
         abi: "llvm".into(),
         linker: Some("clang".into()),
         dynamic_linking: true,
+        dll_tls_export: false,
         dll_prefix: "".into(),
         dll_suffix: ".dll".into(),
         exe_suffix: ".exe".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_pc_nto_qnx710.rs b/compiler/rustc_target/src/spec/x86_64_pc_nto_qnx710.rs
index e9b3acee2..6fb2dfd80 100644
--- a/compiler/rustc_target/src/spec/x86_64_pc_nto_qnx710.rs
+++ b/compiler/rustc_target/src/spec/x86_64_pc_nto_qnx710.rs
@@ -15,6 +15,7 @@ pub fn target() -> Target {
                 LinkerFlavor::Gnu(Cc::Yes, Lld::No),
                 &["-Vgcc_ntox86_64_cxx"],
             ),
+            env: "nto71".into(),
             ..nto_qnx_base::opts()
         },
     }
diff --git a/compiler/rustc_trait_selection/locales/en-US.ftl b/compiler/rustc_trait_selection/messages.ftl
index 14eb4a550..14eb4a550 100644
--- a/compiler/rustc_trait_selection/locales/en-US.ftl
+++ b/compiler/rustc_trait_selection/messages.ftl
diff --git a/compiler/rustc_trait_selection/src/infer.rs b/compiler/rustc_trait_selection/src/infer.rs
index 9b47c7299..911cc0b88 100644
--- a/compiler/rustc_trait_selection/src/infer.rs
+++ b/compiler/rustc_trait_selection/src/infer.rs
@@ -8,26 +8,16 @@ use rustc_middle::infer::canonical::{Canonical, CanonicalQueryResponse, QueryRes
 use rustc_middle::traits::query::Fallible;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_middle::ty::{GenericArg, ToPredicate};
-use rustc_span::{Span, DUMMY_SP};
+use rustc_span::DUMMY_SP;
 
 use std::fmt::Debug;
 
 pub use rustc_infer::infer::*;
 
 pub trait InferCtxtExt<'tcx> {
-    fn type_is_copy_modulo_regions(
-        &self,
-        param_env: ty::ParamEnv<'tcx>,
-        ty: Ty<'tcx>,
-        span: Span,
-    ) -> bool;
+    fn type_is_copy_modulo_regions(&self, param_env: ty::ParamEnv<'tcx>, ty: Ty<'tcx>) -> bool;
 
-    fn type_is_sized_modulo_regions(
-        &self,
-        param_env: ty::ParamEnv<'tcx>,
-        ty: Ty<'tcx>,
-        span: Span,
-    ) -> bool;
+    fn type_is_sized_modulo_regions(&self, param_env: ty::ParamEnv<'tcx>, ty: Ty<'tcx>) -> bool;
 
     /// Check whether a `ty` implements given trait(trait_def_id).
     /// The inputs are:
@@ -46,13 +36,9 @@ pub trait InferCtxtExt<'tcx> {
         param_env: ty::ParamEnv<'tcx>,
     ) -> traits::EvaluationResult;
 }
+
 impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
-    fn type_is_copy_modulo_regions(
-        &self,
-        param_env: ty::ParamEnv<'tcx>,
-        ty: Ty<'tcx>,
-        span: Span,
-    ) -> bool {
+    fn type_is_copy_modulo_regions(&self, param_env: ty::ParamEnv<'tcx>, ty: Ty<'tcx>) -> bool {
         let ty = self.resolve_vars_if_possible(ty);
 
         if !(param_env, ty).needs_infer() {
@@ -65,17 +51,12 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         // rightly refuses to work with inference variables, but
         // moves_by_default has a cache, which we want to use in other
         // cases.
-        traits::type_known_to_meet_bound_modulo_regions(self, param_env, ty, copy_def_id, span)
+        traits::type_known_to_meet_bound_modulo_regions(self, param_env, ty, copy_def_id)
     }
 
-    fn type_is_sized_modulo_regions(
-        &self,
-        param_env: ty::ParamEnv<'tcx>,
-        ty: Ty<'tcx>,
-        span: Span,
-    ) -> bool {
+    fn type_is_sized_modulo_regions(&self, param_env: ty::ParamEnv<'tcx>, ty: Ty<'tcx>) -> bool {
         let lang_item = self.tcx.require_lang_item(LangItem::Sized, None);
-        traits::type_known_to_meet_bound_modulo_regions(self, param_env, ty, lang_item, span)
+        traits::type_known_to_meet_bound_modulo_regions(self, param_env, ty, lang_item)
     }
 
     #[instrument(level = "debug", skip(self, params), ret)]
diff --git a/compiler/rustc_trait_selection/src/lib.rs b/compiler/rustc_trait_selection/src/lib.rs
index 548b42cef..f866cb016 100644
--- a/compiler/rustc_trait_selection/src/lib.rs
+++ b/compiler/rustc_trait_selection/src/lib.rs
@@ -44,4 +44,4 @@ pub mod infer;
 pub mod solve;
 pub mod traits;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_trait_selection/src/solve/assembly.rs b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
index dec9f8016..10d817f75 100644
--- a/compiler/rustc_trait_selection/src/solve/assembly.rs
+++ b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
@@ -1,16 +1,21 @@
 //! Code shared by trait and projection goals for candidate assembly.
 
-#[cfg(doc)]
-use super::trait_goals::structural_traits::*;
-use super::{CanonicalResponse, Certainty, EvalCtxt, Goal, MaybeCause, QueryResult};
-use itertools::Itertools;
+use super::search_graph::OverflowHandler;
+use super::{EvalCtxt, SolverMode};
+use crate::solve::CanonicalResponseExt;
+use crate::traits::coherence;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_hir::def_id::DefId;
 use rustc_infer::traits::query::NoSolution;
-use rustc_infer::traits::util::elaborate_predicates;
+use rustc_infer::traits::util::elaborate;
+use rustc_middle::traits::solve::{CanonicalResponse, Certainty, Goal, MaybeCause, QueryResult};
+use rustc_middle::ty::fast_reject::TreatProjections;
 use rustc_middle::ty::TypeFoldable;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use std::fmt::Debug;
 
+pub(super) mod structural_traits;
+
 /// A candidate is a possible way to prove a goal.
 ///
 /// It consists of both the `source`, which describes how that goal would be proven,
@@ -85,6 +90,8 @@ pub(super) enum CandidateSource {
 pub(super) trait GoalKind<'tcx>: TypeFoldable<TyCtxt<'tcx>> + Copy + Eq {
     fn self_ty(self) -> Ty<'tcx>;
 
+    fn trait_ref(self, tcx: TyCtxt<'tcx>) -> ty::TraitRef<'tcx>;
+
     fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self;
 
     fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId;
@@ -148,6 +155,12 @@ pub(super) trait GoalKind<'tcx>: TypeFoldable<TyCtxt<'tcx>> + Copy + Eq {
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
 
+    // A type is a `FnPtr` if it is of `FnPtr` type.
+    fn consider_builtin_fn_ptr_trait_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
+
     // A callable type (a closure, fn def, or fn ptr) is known to implement the `Fn<A>`
     // family of traits where `A` is given by the signature of the type.
     fn consider_builtin_fn_trait_candidates(
@@ -207,6 +220,16 @@ pub(super) trait GoalKind<'tcx>: TypeFoldable<TyCtxt<'tcx>> + Copy + Eq {
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
+
+    fn consider_builtin_destruct_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
+
+    fn consider_builtin_transmute_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
 }
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
@@ -222,7 +245,9 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         if goal.predicate.self_ty().is_ty_var() {
             return vec![Candidate {
                 source: CandidateSource::BuiltinImpl,
-                result: self.make_canonical_response(Certainty::AMBIGUOUS).unwrap(),
+                result: self
+                    .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
+                    .unwrap(),
             }];
         }
 
@@ -240,14 +265,17 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
 
         self.assemble_object_bound_candidates(goal, &mut candidates);
 
+        self.assemble_coherence_unknowable_candidates(goal, &mut candidates);
+
         candidates
     }
 
     /// If the self type of a goal is a projection, computing the relevant candidates is difficult.
     ///
     /// To deal with this, we first try to normalize the self type and add the candidates for the normalized
-    /// self type to the list of candidates in case that succeeds. Note that we can't just eagerly return in
-    /// this case as projections as self types add `
+    /// self type to the list of candidates in case that succeeds. We also have to consider candidates with the
+    /// projection as a self type as well
+    #[instrument(level = "debug", skip_all)]
     fn assemble_candidates_after_normalizing_self_ty<G: GoalKind<'tcx>>(
         &mut self,
         goal: Goal<'tcx, G>,
@@ -258,48 +286,52 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         let &ty::Alias(ty::Projection, projection_ty) = goal.predicate.self_ty().kind() else {
             return
         };
-        self.probe(|this| {
-            let normalized_ty = this.next_ty_infer();
-            let normalizes_to_goal = goal.with(
-                tcx,
-                ty::Binder::dummy(ty::ProjectionPredicate {
-                    projection_ty,
-                    term: normalized_ty.into(),
-                }),
-            );
-            let normalization_certainty = match this.evaluate_goal(normalizes_to_goal) {
-                Ok((_, certainty)) => certainty,
-                Err(NoSolution) => return,
-            };
-            let normalized_ty = this.resolve_vars_if_possible(normalized_ty);
-
-            // NOTE: Alternatively we could call `evaluate_goal` here and only have a `Normalized` candidate.
-            // This doesn't work as long as we use `CandidateSource` in winnowing.
-            let goal = goal.with(tcx, goal.predicate.with_self_ty(tcx, normalized_ty));
-            let normalized_candidates = this.assemble_and_evaluate_candidates(goal);
-            for mut normalized_candidate in normalized_candidates {
-                normalized_candidate.result =
-                    normalized_candidate.result.unchecked_map(|mut response| {
-                        // FIXME: This currently hides overflow in the normalization step of the self type
-                        // which is probably wrong. Maybe `unify_and` should actually keep overflow as
-                        // we treat it as non-fatal anyways.
-                        response.certainty = response.certainty.unify_and(normalization_certainty);
-                        response
-                    });
-                candidates.push(normalized_candidate);
-            }
-        })
+
+        let normalized_self_candidates: Result<_, NoSolution> = self.probe(|ecx| {
+            ecx.with_incremented_depth(
+                |ecx| {
+                    let result = ecx.evaluate_added_goals_and_make_canonical_response(
+                        Certainty::Maybe(MaybeCause::Overflow),
+                    )?;
+                    Ok(vec![Candidate { source: CandidateSource::BuiltinImpl, result }])
+                },
+                |ecx| {
+                    let normalized_ty = ecx.next_ty_infer();
+                    let normalizes_to_goal = goal.with(
+                        tcx,
+                        ty::Binder::dummy(ty::ProjectionPredicate {
+                            projection_ty,
+                            term: normalized_ty.into(),
+                        }),
+                    );
+                    ecx.add_goal(normalizes_to_goal);
+                    let _ = ecx.try_evaluate_added_goals()?;
+                    let normalized_ty = ecx.resolve_vars_if_possible(normalized_ty);
+                    // NOTE: Alternatively we could call `evaluate_goal` here and only
+                    // have a `Normalized` candidate. This doesn't work as long as we
+                    // use `CandidateSource` in winnowing.
+                    let goal = goal.with(tcx, goal.predicate.with_self_ty(tcx, normalized_ty));
+                    Ok(ecx.assemble_and_evaluate_candidates(goal))
+                },
+            )
+        });
+
+        if let Ok(normalized_self_candidates) = normalized_self_candidates {
+            candidates.extend(normalized_self_candidates);
+        }
     }
 
+    #[instrument(level = "debug", skip_all)]
     fn assemble_impl_candidates<G: GoalKind<'tcx>>(
         &mut self,
         goal: Goal<'tcx, G>,
         candidates: &mut Vec<Candidate<'tcx>>,
     ) {
         let tcx = self.tcx();
-        tcx.for_each_relevant_impl(
+        tcx.for_each_relevant_impl_treating_projections(
             goal.predicate.trait_def_id(tcx),
             goal.predicate.self_ty(),
+            TreatProjections::NextSolverLookup,
             |impl_def_id| match G::consider_impl_candidate(self, goal, impl_def_id) {
                 Ok(result) => candidates
                     .push(Candidate { source: CandidateSource::Impl(impl_def_id), result }),
@@ -308,6 +340,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         );
     }
 
+    #[instrument(level = "debug", skip_all)]
     fn assemble_builtin_impl_candidates<G: GoalKind<'tcx>>(
         &mut self,
         goal: Goal<'tcx, G>,
@@ -315,6 +348,14 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ) {
         let lang_items = self.tcx().lang_items();
         let trait_def_id = goal.predicate.trait_def_id(self.tcx());
+
+        // N.B. When assembling built-in candidates for lang items that are also
+        // `auto` traits, then the auto trait candidate that is assembled in
+        // `consider_auto_trait_candidate` MUST be disqualified to remain sound.
+        //
+        // Instead of adding the logic here, it's a better idea to add it in
+        // `EvalCtxt::disqualify_auto_trait_candidate_due_to_possible_impl` in
+        // `solve::trait_goals` instead.
         let result = if self.tcx().trait_is_auto(trait_def_id) {
             G::consider_auto_trait_candidate(self, goal)
         } else if self.tcx().trait_is_alias(trait_def_id) {
@@ -327,6 +368,8 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             G::consider_builtin_copy_clone_candidate(self, goal)
         } else if lang_items.pointer_like() == Some(trait_def_id) {
             G::consider_builtin_pointer_like_candidate(self, goal)
+        } else if lang_items.fn_ptr_trait() == Some(trait_def_id) {
+            G::consider_builtin_fn_ptr_trait_candidate(self, goal)
         } else if let Some(kind) = self.tcx().fn_trait_kind_from_def_id(trait_def_id) {
             G::consider_builtin_fn_trait_candidates(self, goal, kind)
         } else if lang_items.tuple_trait() == Some(trait_def_id) {
@@ -341,6 +384,10 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             G::consider_builtin_unsize_candidate(self, goal)
         } else if lang_items.discriminant_kind_trait() == Some(trait_def_id) {
             G::consider_builtin_discriminant_kind_candidate(self, goal)
+        } else if lang_items.destruct_trait() == Some(trait_def_id) {
+            G::consider_builtin_destruct_candidate(self, goal)
+        } else if lang_items.transmute_trait() == Some(trait_def_id) {
+            G::consider_builtin_transmute_candidate(self, goal)
         } else {
             Err(NoSolution)
         };
@@ -361,6 +408,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip_all)]
     fn assemble_param_env_candidates<G: GoalKind<'tcx>>(
         &mut self,
         goal: Goal<'tcx, G>,
@@ -376,6 +424,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip_all)]
     fn assemble_alias_bound_candidates<G: GoalKind<'tcx>>(
         &mut self,
         goal: Goal<'tcx, G>,
@@ -423,6 +472,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip_all)]
     fn assemble_object_bound_candidates<G: GoalKind<'tcx>>(
         &mut self,
         goal: Goal<'tcx, G>,
@@ -461,10 +511,25 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         };
 
         let tcx = self.tcx();
-        for assumption in
-            elaborate_predicates(tcx, bounds.iter().map(|bound| bound.with_self_ty(tcx, self_ty)))
+        let own_bounds: FxIndexSet<_> =
+            bounds.iter().map(|bound| bound.with_self_ty(tcx, self_ty)).collect();
+        for assumption in elaborate(tcx, own_bounds.iter().copied())
+            // we only care about bounds that match the `Self` type
+            .filter_only_self()
         {
-            match G::consider_object_bound_candidate(self, goal, assumption.predicate) {
+            // FIXME: Predicates are fully elaborated in the object type's existential bounds
+            // list. We want to only consider these pre-elaborated projections, and not other
+            // projection predicates that we reach by elaborating the principal trait ref,
+            // since that'll cause ambiguity.
+            //
+            // We can remove this when we have implemented intersections in responses.
+            if assumption.to_opt_poly_projection_pred().is_some()
+                && !own_bounds.contains(&assumption)
+            {
+                continue;
+            }
+
+            match G::consider_object_bound_candidate(self, goal, assumption) {
                 Ok(result) => {
                     candidates.push(Candidate { source: CandidateSource::BuiltinImpl, result })
                 }
@@ -473,78 +538,68 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         }
     }
 
-    #[instrument(level = "debug", skip(self), ret)]
-    pub(super) fn merge_candidates_and_discard_reservation_impls(
+    #[instrument(level = "debug", skip_all)]
+    fn assemble_coherence_unknowable_candidates<G: GoalKind<'tcx>>(
         &mut self,
-        mut candidates: Vec<Candidate<'tcx>>,
-    ) -> QueryResult<'tcx> {
-        match candidates.len() {
-            0 => return Err(NoSolution),
-            1 => return Ok(self.discard_reservation_impl(candidates.pop().unwrap()).result),
-            _ => {}
-        }
-
-        if candidates.len() > 1 {
-            let mut i = 0;
-            'outer: while i < candidates.len() {
-                for j in (0..candidates.len()).filter(|&j| i != j) {
-                    if self.trait_candidate_should_be_dropped_in_favor_of(
-                        &candidates[i],
-                        &candidates[j],
-                    ) {
-                        debug!(candidate = ?candidates[i], "Dropping candidate #{}/{}", i, candidates.len());
-                        candidates.swap_remove(i);
-                        continue 'outer;
-                    }
+        goal: Goal<'tcx, G>,
+        candidates: &mut Vec<Candidate<'tcx>>,
+    ) {
+        match self.solver_mode() {
+            SolverMode::Normal => return,
+            SolverMode::Coherence => {
+                let trait_ref = goal.predicate.trait_ref(self.tcx());
+                match coherence::trait_ref_is_knowable(self.tcx(), trait_ref) {
+                    Ok(()) => {}
+                    Err(_) => match self
+                        .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
+                    {
+                        Ok(result) => candidates
+                            .push(Candidate { source: CandidateSource::BuiltinImpl, result }),
+                        // FIXME: This will be reachable at some point if we're in
+                        // `assemble_candidates_after_normalizing_self_ty` and we get a
+                        // universe error. We'll deal with it at this point.
+                        Err(NoSolution) => bug!("coherence candidate resulted in NoSolution"),
+                    },
                 }
-
-                debug!(candidate = ?candidates[i], "Retaining candidate #{}/{}", i, candidates.len());
-                i += 1;
-            }
-
-            // If there are *STILL* multiple candidates that have *different* response
-            // results, give up and report ambiguity.
-            if candidates.len() > 1 && !candidates.iter().map(|cand| cand.result).all_equal() {
-                let certainty = if candidates.iter().all(|x| {
-                    matches!(x.result.value.certainty, Certainty::Maybe(MaybeCause::Overflow))
-                }) {
-                    Certainty::Maybe(MaybeCause::Overflow)
-                } else {
-                    Certainty::AMBIGUOUS
-                };
-                return self.make_canonical_response(certainty);
             }
         }
-
-        // FIXME: What if there are >1 candidates left with the same response, and one is a reservation impl?
-        Ok(self.discard_reservation_impl(candidates.pop().unwrap()).result)
     }
 
-    fn trait_candidate_should_be_dropped_in_favor_of(
-        &self,
-        candidate: &Candidate<'tcx>,
-        other: &Candidate<'tcx>,
-    ) -> bool {
-        // FIXME: implement this
-        match (candidate.source, other.source) {
-            (CandidateSource::Impl(_), _)
-            | (CandidateSource::ParamEnv(_), _)
-            | (CandidateSource::AliasBound, _)
-            | (CandidateSource::BuiltinImpl, _) => false,
+    /// If there are multiple ways to prove a trait or projection goal, we have
+    /// to somehow try to merge the candidates into one. If that fails, we return
+    /// ambiguity.
+    #[instrument(level = "debug", skip(self), ret)]
+    pub(super) fn merge_candidates(
+        &mut self,
+        mut candidates: Vec<Candidate<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        // First try merging all candidates. This is complete and fully sound.
+        let responses = candidates.iter().map(|c| c.result).collect::<Vec<_>>();
+        if let Some(result) = self.try_merge_responses(&responses) {
+            return Ok(result);
         }
-    }
 
-    fn discard_reservation_impl(&self, mut candidate: Candidate<'tcx>) -> Candidate<'tcx> {
-        if let CandidateSource::Impl(def_id) = candidate.source {
-            if let ty::ImplPolarity::Reservation = self.tcx().impl_polarity(def_id) {
-                debug!("Selected reservation impl");
-                // We assemble all candidates inside of a probe so by
-                // making a new canonical response here our result will
-                // have no constraints.
-                candidate.result = self.make_canonical_response(Certainty::AMBIGUOUS).unwrap();
+        // We then check whether we should prioritize `ParamEnv` candidates.
+        //
+        // Doing so is incomplete and would therefore be unsound during coherence.
+        match self.solver_mode() {
+            SolverMode::Coherence => (),
+            // Prioritize `ParamEnv` candidates only if they do not guide inference.
+            //
+            // This is still incomplete as we may add incorrect region bounds.
+            SolverMode::Normal => {
+                let param_env_responses = candidates
+                    .iter()
+                    .filter(|c| matches!(c.source, CandidateSource::ParamEnv(_)))
+                    .map(|c| c.result)
+                    .collect::<Vec<_>>();
+                if let Some(result) = self.try_merge_responses(&param_env_responses) {
+                    if result.has_only_region_constraints() {
+                        return Ok(result);
+                    }
+                }
             }
         }
-
-        candidate
+        self.flounder(&responses)
     }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals/structural_traits.rs b/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs
index d7d93377c..1a566e87d 100644
--- a/compiler/rustc_trait_selection/src/solve/trait_goals/structural_traits.rs
+++ b/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs
@@ -1,7 +1,9 @@
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::{def_id::DefId, Movability, Mutability};
 use rustc_infer::traits::query::NoSolution;
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable};
+use rustc_middle::ty::{
+    self, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
+};
 
 use crate::solve::EvalCtxt;
 
@@ -9,7 +11,7 @@ use crate::solve::EvalCtxt;
 //
 // For types with an "existential" binder, i.e. generator witnesses, we also
 // instantiate the binder with placeholders eagerly.
-pub(super) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
+pub(in crate::solve) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
     ecx: &EvalCtxt<'_, 'tcx>,
     ty: Ty<'tcx>,
 ) -> Result<Vec<Ty<'tcx>>, NoSolution> {
@@ -22,21 +24,19 @@ pub(super) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
         | ty::FnDef(..)
         | ty::FnPtr(_)
         | ty::Error(_)
-        | ty::Infer(ty::IntVar(_) | ty::FloatVar(_))
         | ty::Never
         | ty::Char => Ok(vec![]),
 
-        // Treat this like `struct str([u8]);`
+        // Treat `str` like it's defined as `struct str([u8]);`
         ty::Str => Ok(vec![tcx.mk_slice(tcx.types.u8)]),
 
         ty::Dynamic(..)
         | ty::Param(..)
         | ty::Foreign(..)
         | ty::Alias(ty::Projection, ..)
-        | ty::Placeholder(..) => Err(NoSolution),
-
-        ty::Bound(..)
-        | ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
+        | ty::Placeholder(..)
+        | ty::Bound(..)
+        | ty::Infer(_) => {
             bug!("unexpected type `{ty}`")
         }
 
@@ -60,7 +60,16 @@ pub(super) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
 
         ty::GeneratorWitness(types) => Ok(ecx.instantiate_binder_with_placeholders(types).to_vec()),
 
-        ty::GeneratorWitnessMIR(..) => todo!(),
+        ty::GeneratorWitnessMIR(def_id, substs) => Ok(ecx
+            .tcx()
+            .generator_hidden_types(def_id)
+            .map(|bty| {
+                ecx.instantiate_binder_with_placeholders(replace_erased_lifetimes_with_bound_vars(
+                    tcx,
+                    bty.subst(tcx, substs),
+                ))
+            })
+            .collect()),
 
         // For `PhantomData<T>`, we pass `T`.
         ty::Adt(def, substs) if def.is_phantom_data() => Ok(vec![substs.type_at(0)]),
@@ -76,7 +85,28 @@ pub(super) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
     }
 }
 
-pub(super) fn instantiate_constituent_tys_for_sized_trait<'tcx>(
+pub(in crate::solve) fn replace_erased_lifetimes_with_bound_vars<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    ty: Ty<'tcx>,
+) -> ty::Binder<'tcx, Ty<'tcx>> {
+    debug_assert!(!ty.has_late_bound_regions());
+    let mut counter = 0;
+    let ty = tcx.fold_regions(ty, |mut r, current_depth| {
+        if let ty::ReErased = r.kind() {
+            let br =
+                ty::BoundRegion { var: ty::BoundVar::from_u32(counter), kind: ty::BrAnon(None) };
+            counter += 1;
+            r = tcx.mk_re_late_bound(current_depth, br);
+        }
+        r
+    });
+    let bound_vars = tcx.mk_bound_variable_kinds_from_iter(
+        (0..counter).map(|_| ty::BoundVariableKind::Region(ty::BrAnon(None))),
+    );
+    ty::Binder::bind_with_vars(ty, bound_vars)
+}
+
+pub(in crate::solve) fn instantiate_constituent_tys_for_sized_trait<'tcx>(
     ecx: &EvalCtxt<'_, 'tcx>,
     ty: Ty<'tcx>,
 ) -> Result<Vec<Ty<'tcx>>, NoSolution> {
@@ -126,7 +156,7 @@ pub(super) fn instantiate_constituent_tys_for_sized_trait<'tcx>(
     }
 }
 
-pub(super) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>(
+pub(in crate::solve) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>(
     ecx: &EvalCtxt<'_, 'tcx>,
     ty: Ty<'tcx>,
 ) -> Result<Vec<Ty<'tcx>>, NoSolution> {
@@ -178,29 +208,65 @@ pub(super) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>(
 
         ty::GeneratorWitness(types) => Ok(ecx.instantiate_binder_with_placeholders(types).to_vec()),
 
-        ty::GeneratorWitnessMIR(..) => todo!(),
+        ty::GeneratorWitnessMIR(def_id, substs) => Ok(ecx
+            .tcx()
+            .generator_hidden_types(def_id)
+            .map(|bty| {
+                ecx.instantiate_binder_with_placeholders(replace_erased_lifetimes_with_bound_vars(
+                    ecx.tcx(),
+                    bty.subst(ecx.tcx(), substs),
+                ))
+            })
+            .collect()),
     }
 }
 
 // Returns a binder of the tupled inputs types and output type from a builtin callable type.
-pub(crate) fn extract_tupled_inputs_and_output_from_callable<'tcx>(
+pub(in crate::solve) fn extract_tupled_inputs_and_output_from_callable<'tcx>(
     tcx: TyCtxt<'tcx>,
     self_ty: Ty<'tcx>,
     goal_kind: ty::ClosureKind,
 ) -> Result<Option<ty::Binder<'tcx, (Ty<'tcx>, Ty<'tcx>)>>, NoSolution> {
     match *self_ty.kind() {
-        ty::FnDef(def_id, substs) => Ok(Some(
-            tcx.fn_sig(def_id)
-                .subst(tcx, substs)
-                .map_bound(|sig| (tcx.mk_tup(sig.inputs()), sig.output())),
-        )),
-        ty::FnPtr(sig) => Ok(Some(sig.map_bound(|sig| (tcx.mk_tup(sig.inputs()), sig.output())))),
+        // keep this in sync with assemble_fn_pointer_candidates until the old solver is removed.
+        ty::FnDef(def_id, substs) => {
+            let sig = tcx.fn_sig(def_id);
+            if sig.skip_binder().is_fn_trait_compatible()
+                && tcx.codegen_fn_attrs(def_id).target_features.is_empty()
+            {
+                Ok(Some(
+                    sig.subst(tcx, substs)
+                        .map_bound(|sig| (tcx.mk_tup(sig.inputs()), sig.output())),
+                ))
+            } else {
+                Err(NoSolution)
+            }
+        }
+        // keep this in sync with assemble_fn_pointer_candidates until the old solver is removed.
+        ty::FnPtr(sig) => {
+            if sig.is_fn_trait_compatible() {
+                Ok(Some(sig.map_bound(|sig| (tcx.mk_tup(sig.inputs()), sig.output()))))
+            } else {
+                Err(NoSolution)
+            }
+        }
         ty::Closure(_, substs) => {
             let closure_substs = substs.as_closure();
             match closure_substs.kind_ty().to_opt_closure_kind() {
-                Some(closure_kind) if closure_kind.extends(goal_kind) => {}
-                None => return Ok(None),
-                _ => return Err(NoSolution),
+                // If the closure's kind doesn't extend the goal kind,
+                // then the closure doesn't implement the trait.
+                Some(closure_kind) => {
+                    if !closure_kind.extends(goal_kind) {
+                        return Err(NoSolution);
+                    }
+                }
+                // Closure kind is not yet determined, so we return ambiguity unless
+                // the expected kind is `FnOnce` as that is always implemented.
+                None => {
+                    if goal_kind != ty::ClosureKind::FnOnce {
+                        return Ok(None);
+                    }
+                }
             }
             Ok(Some(closure_substs.sig().map_bound(|sig| (sig.inputs()[0], sig.output()))))
         }
@@ -269,7 +335,13 @@ pub(crate) fn extract_tupled_inputs_and_output_from_callable<'tcx>(
 /// additional step of eagerly folding the associated types in the where
 /// clauses of the impl. In this example, that means replacing
 /// `<Self as Foo>::Bar` with `Ty` in the first impl.
-pub(crate) fn predicates_for_object_candidate<'tcx>(
+///
+// FIXME: This is only necessary as `<Self as Trait>::Assoc: ItemBound`
+// bounds in impls are trivially proven using the item bound candidates.
+// This is unsound in general and once that is fixed, we don't need to
+// normalize eagerly here. See https://github.com/lcnr/solver-woes/issues/9
+// for more details.
+pub(in crate::solve) fn predicates_for_object_candidate<'tcx>(
     ecx: &EvalCtxt<'_, 'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     trait_ref: ty::TraitRef<'tcx>,
@@ -333,7 +405,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReplaceProjectionWith<'_, 'tcx> {
             // FIXME: Technically this folder could be fallible?
             let nested = self
                 .ecx
-                .eq(self.param_env, alias_ty, proj.projection_ty)
+                .eq_and_get_goals(self.param_env, alias_ty, proj.projection_ty)
                 .expect("expected to be able to unify goal projection with dyn's projection");
             // FIXME: Technically we could register these too..
             assert!(nested.is_empty(), "did not expect unification to have any nested goals");
diff --git a/compiler/rustc_trait_selection/src/solve/canonical/canonicalize.rs b/compiler/rustc_trait_selection/src/solve/canonicalize.rs
index c048d4a2a..976849696 100644
--- a/compiler/rustc_trait_selection/src/solve/canonical/canonicalize.rs
+++ b/compiler/rustc_trait_selection/src/solve/canonicalize.rs
@@ -13,7 +13,7 @@ use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Ty};
 use rustc_middle::ty::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 
-/// Whether we're canonicalizing a query input or the query reponse.
+/// Whether we're canonicalizing a query input or the query response.
 ///
 /// When canonicalizing an input we're in the context of the caller
 /// while canonicalizing the response happens in the context of the
@@ -21,7 +21,7 @@ use rustc_middle::ty::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 #[derive(Debug, Clone, Copy)]
 pub enum CanonicalizeMode {
     Input,
-    /// FIXME: We currently return region constraints refering to
+    /// FIXME: We currently return region constraints referring to
     /// placeholders and inference variables from a binder instantiated
     /// inside of the query.
     ///
@@ -125,8 +125,9 @@ impl<'a, 'tcx> Canonicalizer<'a, 'tcx> {
         // - var_infos: [E0, U1, E1, U1, E1, E6, U6], curr_compressed_uv: 1, next_orig_uv: 6
         // - var_infos: [E0, U1, E1, U1, E1, E2, U2], curr_compressed_uv: 2, next_orig_uv: -
         //
-        // This algorithm runs in `O(n²)` where `n` is the number of different universe
-        // indices in the input. This should be fine as `n` is expected to be small.
+        // This algorithm runs in `O(nm)` where `n` is the number of different universe
+        // indices in the input and `m` is the number of canonical variables.
+        // This should be fine as both `n` and `m` are expected to be small.
         let mut curr_compressed_uv = ty::UniverseIndex::ROOT;
         let mut existential_in_new_uv = false;
         let mut next_orig_uv = Some(ty::UniverseIndex::ROOT);
@@ -245,42 +246,49 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
             ty::ReError(_) => return r,
         };
 
-        let existing_bound_var = match self.canonicalize_mode {
-            CanonicalizeMode::Input => None,
-            CanonicalizeMode::Response { .. } => {
-                self.variables.iter().position(|&v| v == r.into()).map(ty::BoundVar::from)
-            }
-        };
-        let var = existing_bound_var.unwrap_or_else(|| {
-            let var = ty::BoundVar::from(self.variables.len());
-            self.variables.push(r.into());
-            self.primitive_var_infos.push(CanonicalVarInfo { kind });
-            var
-        });
-        let br = ty::BoundRegion { var, kind: BrAnon(var.as_u32(), None) };
+        let var = ty::BoundVar::from(
+            self.variables.iter().position(|&v| v == r.into()).unwrap_or_else(|| {
+                let var = self.variables.len();
+                self.variables.push(r.into());
+                self.primitive_var_infos.push(CanonicalVarInfo { kind });
+                var
+            }),
+        );
+        let br = ty::BoundRegion { var, kind: BrAnon(None) };
         self.interner().mk_re_late_bound(self.binder_index, br)
     }
 
-    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
+    fn fold_ty(&mut self, mut t: Ty<'tcx>) -> Ty<'tcx> {
         let kind = match *t.kind() {
-            ty::Infer(ty::TyVar(vid)) => match self.infcx.probe_ty_var(vid) {
-                Ok(t) => return self.fold_ty(t),
-                Err(ui) => CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)),
-            },
-            ty::Infer(ty::IntVar(_)) => {
-                let nt = self.infcx.shallow_resolve(t);
+            ty::Infer(ty::TyVar(mut vid)) => {
+                // We need to canonicalize the *root* of our ty var.
+                // This is so that our canonical response correctly reflects
+                // any equated inference vars correctly!
+                let root_vid = self.infcx.root_var(vid);
+                if root_vid != vid {
+                    t = self.infcx.tcx.mk_ty_var(root_vid);
+                    vid = root_vid;
+                }
+
+                match self.infcx.probe_ty_var(vid) {
+                    Ok(t) => return self.fold_ty(t),
+                    Err(ui) => CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)),
+                }
+            }
+            ty::Infer(ty::IntVar(vid)) => {
+                let nt = self.infcx.opportunistic_resolve_int_var(vid);
                 if nt != t {
                     return self.fold_ty(nt);
                 } else {
                     CanonicalVarKind::Ty(CanonicalTyVarKind::Int)
                 }
             }
-            ty::Infer(ty::FloatVar(_)) => {
-                let nt = self.infcx.shallow_resolve(t);
+            ty::Infer(ty::FloatVar(vid)) => {
+                let nt = self.infcx.opportunistic_resolve_float_var(vid);
                 if nt != t {
                     return self.fold_ty(nt);
                 } else {
-                    CanonicalVarKind::Ty(CanonicalTyVarKind::Int)
+                    CanonicalVarKind::Ty(CanonicalTyVarKind::Float)
                 }
             }
             ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
@@ -289,14 +297,20 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
             ty::Placeholder(placeholder) => match self.canonicalize_mode {
                 CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(ty::Placeholder {
                     universe: placeholder.universe,
-                    name: BoundTyKind::Anon(self.variables.len() as u32),
+                    bound: ty::BoundTy {
+                        var: ty::BoundVar::from_usize(self.variables.len()),
+                        kind: ty::BoundTyKind::Anon,
+                    },
                 }),
                 CanonicalizeMode::Response { .. } => CanonicalVarKind::PlaceholderTy(placeholder),
             },
             ty::Param(_) => match self.canonicalize_mode {
                 CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(ty::Placeholder {
                     universe: ty::UniverseIndex::ROOT,
-                    name: ty::BoundTyKind::Anon(self.variables.len() as u32),
+                    bound: ty::BoundTy {
+                        var: ty::BoundVar::from_usize(self.variables.len()),
+                        kind: ty::BoundTyKind::Anon,
+                    },
                 }),
                 CanonicalizeMode::Response { .. } => bug!("param ty in response: {t:?}"),
             },
@@ -334,17 +348,27 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
                 var
             }),
         );
-        let bt = ty::BoundTy { var, kind: BoundTyKind::Anon(var.index() as u32) };
+        let bt = ty::BoundTy { var, kind: BoundTyKind::Anon };
         self.interner().mk_bound(self.binder_index, bt)
     }
 
-    fn fold_const(&mut self, c: ty::Const<'tcx>) -> ty::Const<'tcx> {
+    fn fold_const(&mut self, mut c: ty::Const<'tcx>) -> ty::Const<'tcx> {
         let kind = match c.kind() {
-            ty::ConstKind::Infer(ty::InferConst::Var(vid)) => match self.infcx.probe_const_var(vid)
-            {
-                Ok(c) => return self.fold_const(c),
-                Err(universe) => CanonicalVarKind::Const(universe, c.ty()),
-            },
+            ty::ConstKind::Infer(ty::InferConst::Var(mut vid)) => {
+                // We need to canonicalize the *root* of our const var.
+                // This is so that our canonical response correctly reflects
+                // any equated inference vars correctly!
+                let root_vid = self.infcx.root_const_var(vid);
+                if root_vid != vid {
+                    c = self.infcx.tcx.mk_const(ty::InferConst::Var(root_vid), c.ty());
+                    vid = root_vid;
+                }
+
+                match self.infcx.probe_const_var(vid) {
+                    Ok(c) => return self.fold_const(c),
+                    Err(universe) => CanonicalVarKind::Const(universe, c.ty()),
+                }
+            }
             ty::ConstKind::Infer(ty::InferConst::Fresh(_)) => {
                 bug!("fresh var during canonicalization: {c:?}")
             }
@@ -352,7 +376,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
                 CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
                     ty::Placeholder {
                         universe: placeholder.universe,
-                        name: ty::BoundVar::from(self.variables.len()),
+                        bound: ty::BoundVar::from(self.variables.len()),
                     },
                     c.ty(),
                 ),
@@ -364,7 +388,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
                 CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
                     ty::Placeholder {
                         universe: ty::UniverseIndex::ROOT,
-                        name: ty::BoundVar::from(self.variables.len()),
+                        bound: ty::BoundVar::from(self.variables.len()),
                     },
                     c.ty(),
                 ),
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs
index 95612674e..c29b5b04e 100644
--- a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs
@@ -2,10 +2,13 @@ use rustc_hir::def_id::DefId;
 use rustc_infer::infer::at::ToTrace;
 use rustc_infer::infer::canonical::CanonicalVarValues;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use rustc_infer::infer::{InferCtxt, InferOk, LateBoundRegionConversionTime};
+use rustc_infer::infer::{
+    DefineOpaqueTypes, InferCtxt, InferOk, LateBoundRegionConversionTime, TyCtxtInferExt,
+};
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::ObligationCause;
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
+use rustc_middle::traits::solve::{CanonicalGoal, Certainty, MaybeCause, QueryResult};
 use rustc_middle::ty::{
     self, Ty, TyCtxt, TypeFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
     TypeVisitor,
@@ -13,12 +16,32 @@ use rustc_middle::ty::{
 use rustc_span::DUMMY_SP;
 use std::ops::ControlFlow;
 
-use super::search_graph::SearchGraph;
-use super::Goal;
+use crate::traits::specialization_graph;
+
+use super::search_graph::{self, OverflowHandler};
+use super::SolverMode;
+use super::{search_graph::SearchGraph, Goal};
+
+mod canonical;
 
 pub struct EvalCtxt<'a, 'tcx> {
-    // FIXME: should be private.
-    pub(super) infcx: &'a InferCtxt<'tcx>,
+    /// The inference context that backs (mostly) inference and placeholder terms
+    /// instantiated while solving goals.
+    ///
+    /// NOTE: The `InferCtxt` that backs the `EvalCtxt` is intentionally private,
+    /// because the `InferCtxt` is much more general than `EvalCtxt`. Methods such
+    /// as  `take_registered_region_obligations` can mess up query responses,
+    /// using `At::normalize` is totally wrong, calling `evaluate_root_goal` can
+    /// cause coinductive unsoundness, etc.
+    ///
+    /// Methods that are generally of use for trait solving are *intentionally*
+    /// re-declared through the `EvalCtxt` below, often with cleaner signatures
+    /// since we don't care about things like `ObligationCause`s and `Span`s here.
+    /// If some `InferCtxt` method is missing, please first think defensively about
+    /// the method's compatibility with this solver, or if an existing one does
+    /// the job already.
+    infcx: &'a InferCtxt<'tcx>,
+
     pub(super) var_values: CanonicalVarValues<'tcx>,
     /// The highest universe index nameable by the caller.
     ///
@@ -33,14 +56,356 @@ pub struct EvalCtxt<'a, 'tcx> {
 
     pub(super) search_graph: &'a mut SearchGraph<'tcx>,
 
-    /// This field is used by a debug assertion in [`EvalCtxt::evaluate_goal`],
-    /// see the comment in that method for more details.
-    pub in_projection_eq_hack: bool,
+    pub(super) nested_goals: NestedGoals<'tcx>,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(super) enum IsNormalizesToHack {
+    Yes,
+    No,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct NestedGoals<'tcx> {
+    /// This normalizes-to goal that is treated specially during the evaluation
+    /// loop. In each iteration we take the RHS of the projection, replace it with
+    /// a fresh inference variable, and only after evaluating that goal do we
+    /// equate the fresh inference variable with the actual RHS of the predicate.
+    ///
+    /// This is both to improve caching, and to avoid using the RHS of the
+    /// projection predicate to influence the normalizes-to candidate we select.
+    ///
+    /// This is not a 'real' nested goal. We must not forget to replace the RHS
+    /// with a fresh inference variable when we evaluate this goal. That can result
+    /// in a trait solver cycle. This would currently result in overflow but can be
+    /// can be unsound with more powerful coinduction in the future.
+    pub(super) normalizes_to_hack_goal: Option<Goal<'tcx, ty::ProjectionPredicate<'tcx>>>,
+    /// The rest of the goals which have not yet processed or remain ambiguous.
+    pub(super) goals: Vec<Goal<'tcx, ty::Predicate<'tcx>>>,
+}
+
+impl NestedGoals<'_> {
+    pub(super) fn new() -> Self {
+        Self { normalizes_to_hack_goal: None, goals: Vec::new() }
+    }
+
+    pub(super) fn is_empty(&self) -> bool {
+        self.normalizes_to_hack_goal.is_none() && self.goals.is_empty()
+    }
+}
+
+pub trait InferCtxtEvalExt<'tcx> {
+    /// Evaluates a goal from **outside** of the trait solver.
+    ///
+    /// Using this while inside of the solver is wrong as it uses a new
+    /// search graph which would break cycle detection.
+    fn evaluate_root_goal(
+        &self,
+        goal: Goal<'tcx, ty::Predicate<'tcx>>,
+    ) -> Result<(bool, Certainty, Vec<Goal<'tcx, ty::Predicate<'tcx>>>), NoSolution>;
+}
+
+impl<'tcx> InferCtxtEvalExt<'tcx> for InferCtxt<'tcx> {
+    #[instrument(level = "debug", skip(self), ret)]
+    fn evaluate_root_goal(
+        &self,
+        goal: Goal<'tcx, ty::Predicate<'tcx>>,
+    ) -> Result<(bool, Certainty, Vec<Goal<'tcx, ty::Predicate<'tcx>>>), NoSolution> {
+        let mode = if self.intercrate { SolverMode::Coherence } else { SolverMode::Normal };
+        let mut search_graph = search_graph::SearchGraph::new(self.tcx, mode);
+
+        let mut ecx = EvalCtxt {
+            search_graph: &mut search_graph,
+            infcx: self,
+            // Only relevant when canonicalizing the response.
+            max_input_universe: ty::UniverseIndex::ROOT,
+            var_values: CanonicalVarValues::dummy(),
+            nested_goals: NestedGoals::new(),
+        };
+        let result = ecx.evaluate_goal(IsNormalizesToHack::No, goal);
+
+        assert!(
+            ecx.nested_goals.is_empty(),
+            "root `EvalCtxt` should not have any goals added to it"
+        );
+
+        assert!(search_graph.is_empty());
+        result
+    }
+}
+
+impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
+    pub(super) fn solver_mode(&self) -> SolverMode {
+        self.search_graph.solver_mode()
+    }
+
+    /// The entry point of the solver.
+    ///
+    /// This function deals with (coinductive) cycles, overflow, and caching
+    /// and then calls [`EvalCtxt::compute_goal`] which contains the actual
+    /// logic of the solver.
+    ///
+    /// Instead of calling this function directly, use either [EvalCtxt::evaluate_goal]
+    /// if you're inside of the solver or [InferCtxtEvalExt::evaluate_root_goal] if you're
+    /// outside of it.
+    #[instrument(level = "debug", skip(tcx, search_graph), ret)]
+    fn evaluate_canonical_goal(
+        tcx: TyCtxt<'tcx>,
+        search_graph: &'a mut search_graph::SearchGraph<'tcx>,
+        canonical_goal: CanonicalGoal<'tcx>,
+    ) -> QueryResult<'tcx> {
+        // Deal with overflow, caching, and coinduction.
+        //
+        // The actual solver logic happens in `ecx.compute_goal`.
+        search_graph.with_new_goal(tcx, canonical_goal, |search_graph| {
+            let intercrate = match search_graph.solver_mode() {
+                SolverMode::Normal => false,
+                SolverMode::Coherence => true,
+            };
+            let (ref infcx, goal, var_values) = tcx
+                .infer_ctxt()
+                .intercrate(intercrate)
+                .build_with_canonical(DUMMY_SP, &canonical_goal);
+            let mut ecx = EvalCtxt {
+                infcx,
+                var_values,
+                max_input_universe: canonical_goal.max_universe,
+                search_graph,
+                nested_goals: NestedGoals::new(),
+            };
+            ecx.compute_goal(goal)
+        })
+    }
+
+    /// Recursively evaluates `goal`, returning whether any inference vars have
+    /// been constrained and the certainty of the result.
+    fn evaluate_goal(
+        &mut self,
+        is_normalizes_to_hack: IsNormalizesToHack,
+        goal: Goal<'tcx, ty::Predicate<'tcx>>,
+    ) -> Result<(bool, Certainty, Vec<Goal<'tcx, ty::Predicate<'tcx>>>), NoSolution> {
+        let (orig_values, canonical_goal) = self.canonicalize_goal(goal);
+        let canonical_response =
+            EvalCtxt::evaluate_canonical_goal(self.tcx(), self.search_graph, canonical_goal)?;
+
+        let has_changed = !canonical_response.value.var_values.is_identity();
+        let (certainty, nested_goals) = self.instantiate_and_apply_query_response(
+            goal.param_env,
+            orig_values,
+            canonical_response,
+        )?;
+
+        if !has_changed && !nested_goals.is_empty() {
+            bug!("an unchanged goal shouldn't have any side-effects on instantiation");
+        }
+
+        // Check that rerunning this query with its inference constraints applied
+        // doesn't result in new inference constraints and has the same result.
+        //
+        // If we have projection goals like `<T as Trait>::Assoc == u32` we recursively
+        // call `exists<U> <T as Trait>::Assoc == U` to enable better caching. This goal
+        // could constrain `U` to `u32` which would cause this check to result in a
+        // solver cycle.
+        if cfg!(debug_assertions)
+            && has_changed
+            && is_normalizes_to_hack == IsNormalizesToHack::No
+            && !self.search_graph.in_cycle()
+        {
+            debug!("rerunning goal to check result is stable");
+            let (_orig_values, canonical_goal) = self.canonicalize_goal(goal);
+            let canonical_response =
+                EvalCtxt::evaluate_canonical_goal(self.tcx(), self.search_graph, canonical_goal)?;
+            if !canonical_response.value.var_values.is_identity() {
+                bug!(
+                    "unstable result: re-canonicalized goal={canonical_goal:#?} \
+                     response={canonical_response:#?}"
+                );
+            }
+            if certainty != canonical_response.value.certainty {
+                bug!(
+                    "unstable certainty: {certainty:#?} re-canonicalized goal={canonical_goal:#?} \
+                     response={canonical_response:#?}"
+                );
+            }
+        }
+
+        Ok((has_changed, certainty, nested_goals))
+    }
+
+    fn compute_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) -> QueryResult<'tcx> {
+        let Goal { param_env, predicate } = goal;
+        let kind = predicate.kind();
+        if let Some(kind) = kind.no_bound_vars() {
+            match kind {
+                ty::PredicateKind::Clause(ty::Clause::Trait(predicate)) => {
+                    self.compute_trait_goal(Goal { param_env, predicate })
+                }
+                ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) => {
+                    self.compute_projection_goal(Goal { param_env, predicate })
+                }
+                ty::PredicateKind::Clause(ty::Clause::TypeOutlives(predicate)) => {
+                    self.compute_type_outlives_goal(Goal { param_env, predicate })
+                }
+                ty::PredicateKind::Clause(ty::Clause::RegionOutlives(predicate)) => {
+                    self.compute_region_outlives_goal(Goal { param_env, predicate })
+                }
+                ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
+                    self.compute_const_arg_has_type_goal(Goal { param_env, predicate: (ct, ty) })
+                }
+                ty::PredicateKind::Subtype(predicate) => {
+                    self.compute_subtype_goal(Goal { param_env, predicate })
+                }
+                ty::PredicateKind::Coerce(predicate) => {
+                    self.compute_coerce_goal(Goal { param_env, predicate })
+                }
+                ty::PredicateKind::ClosureKind(def_id, substs, kind) => self
+                    .compute_closure_kind_goal(Goal {
+                        param_env,
+                        predicate: (def_id, substs, kind),
+                    }),
+                ty::PredicateKind::ObjectSafe(trait_def_id) => {
+                    self.compute_object_safe_goal(trait_def_id)
+                }
+                ty::PredicateKind::WellFormed(arg) => {
+                    self.compute_well_formed_goal(Goal { param_env, predicate: arg })
+                }
+                ty::PredicateKind::Ambiguous => {
+                    self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
+                }
+                // FIXME: implement these predicates :)
+                ty::PredicateKind::ConstEvaluatable(_) | ty::PredicateKind::ConstEquate(_, _) => {
+                    self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+                }
+                ty::PredicateKind::TypeWellFormedFromEnv(..) => {
+                    bug!("TypeWellFormedFromEnv is only used for Chalk")
+                }
+                ty::PredicateKind::AliasRelate(lhs, rhs, direction) => self
+                    .compute_alias_relate_goal(Goal {
+                        param_env,
+                        predicate: (lhs, rhs, direction),
+                    }),
+            }
+        } else {
+            let kind = self.infcx.instantiate_binder_with_placeholders(kind);
+            let goal = goal.with(self.tcx(), ty::Binder::dummy(kind));
+            self.add_goal(goal);
+            self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        }
+    }
+
+    // Recursively evaluates all the goals added to this `EvalCtxt` to completion, returning
+    // the certainty of all the goals.
+    #[instrument(level = "debug", skip(self))]
+    pub(super) fn try_evaluate_added_goals(&mut self) -> Result<Certainty, NoSolution> {
+        let mut goals = core::mem::replace(&mut self.nested_goals, NestedGoals::new());
+        let mut new_goals = NestedGoals::new();
+
+        let response = self.repeat_while_none(
+            |_| Ok(Certainty::Maybe(MaybeCause::Overflow)),
+            |this| {
+                let mut has_changed = Err(Certainty::Yes);
+
+                if let Some(goal) = goals.normalizes_to_hack_goal.take() {
+                    // Replace the goal with an unconstrained infer var, so the
+                    // RHS does not affect projection candidate assembly.
+                    let unconstrained_rhs = this.next_term_infer_of_kind(goal.predicate.term);
+                    let unconstrained_goal = goal.with(
+                        this.tcx(),
+                        ty::Binder::dummy(ty::ProjectionPredicate {
+                            projection_ty: goal.predicate.projection_ty,
+                            term: unconstrained_rhs,
+                        }),
+                    );
+
+                    let (_, certainty, instantiate_goals) =
+                        match this.evaluate_goal(IsNormalizesToHack::Yes, unconstrained_goal) {
+                            Ok(r) => r,
+                            Err(NoSolution) => return Some(Err(NoSolution)),
+                        };
+                    new_goals.goals.extend(instantiate_goals);
+
+                    // Finally, equate the goal's RHS with the unconstrained var.
+                    // We put the nested goals from this into goals instead of
+                    // next_goals to avoid needing to process the loop one extra
+                    // time if this goal returns something -- I don't think this
+                    // matters in practice, though.
+                    match this.eq_and_get_goals(
+                        goal.param_env,
+                        goal.predicate.term,
+                        unconstrained_rhs,
+                    ) {
+                        Ok(eq_goals) => {
+                            goals.goals.extend(eq_goals);
+                        }
+                        Err(NoSolution) => return Some(Err(NoSolution)),
+                    };
+
+                    // We only look at the `projection_ty` part here rather than
+                    // looking at the "has changed" return from evaluate_goal,
+                    // because we expect the `unconstrained_rhs` part of the predicate
+                    // to have changed -- that means we actually normalized successfully!
+                    if goal.predicate.projection_ty
+                        != this.resolve_vars_if_possible(goal.predicate.projection_ty)
+                    {
+                        has_changed = Ok(())
+                    }
+
+                    match certainty {
+                        Certainty::Yes => {}
+                        Certainty::Maybe(_) => {
+                            // We need to resolve vars here so that we correctly
+                            // deal with `has_changed` in the next iteration.
+                            new_goals.normalizes_to_hack_goal =
+                                Some(this.resolve_vars_if_possible(goal));
+                            has_changed = has_changed.map_err(|c| c.unify_with(certainty));
+                        }
+                    }
+                }
+
+                for goal in goals.goals.drain(..) {
+                    let (changed, certainty, instantiate_goals) =
+                        match this.evaluate_goal(IsNormalizesToHack::No, goal) {
+                            Ok(result) => result,
+                            Err(NoSolution) => return Some(Err(NoSolution)),
+                        };
+                    new_goals.goals.extend(instantiate_goals);
+
+                    if changed {
+                        has_changed = Ok(());
+                    }
+
+                    match certainty {
+                        Certainty::Yes => {}
+                        Certainty::Maybe(_) => {
+                            new_goals.goals.push(goal);
+                            has_changed = has_changed.map_err(|c| c.unify_with(certainty));
+                        }
+                    }
+                }
+
+                core::mem::swap(&mut new_goals, &mut goals);
+                match has_changed {
+                    Ok(()) => None,
+                    Err(certainty) => Some(Ok(certainty)),
+                }
+            },
+        );
+
+        self.nested_goals = goals;
+        response
+    }
 }
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     pub(super) fn probe<T>(&mut self, f: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> T) -> T {
-        self.infcx.probe(|_| f(self))
+        let mut ecx = EvalCtxt {
+            infcx: self.infcx,
+            var_values: self.var_values,
+            max_input_universe: self.max_input_universe,
+            search_graph: self.search_graph,
+            nested_goals: self.nested_goals.clone(),
+        };
+        self.infcx.probe(|_| f(&mut ecx))
     }
 
     pub(super) fn tcx(&self) -> TyCtxt<'tcx> {
@@ -61,6 +426,15 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         )
     }
 
+    /// Returns a ty infer or a const infer depending on whether `kind` is a `Ty` or `Const`.
+    /// If `kind` is an integer inference variable this will still return a ty infer var.
+    pub(super) fn next_term_infer_of_kind(&self, kind: ty::Term<'tcx>) -> ty::Term<'tcx> {
+        match kind.unpack() {
+            ty::TermKind::Ty(_) => self.next_ty_infer().into(),
+            ty::TermKind::Const(ct) => self.next_const_infer(ct.ty()).into(),
+        }
+    }
+
     /// Is the projection predicate is of the form `exists<T> <Ty as Trait>::Assoc = T`.
     ///
     /// This is the case if the `term` is an inference variable in the innermost universe
@@ -74,7 +448,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
                 if let &ty::Infer(ty::TyVar(vid)) = ty.kind() {
                     match self.infcx.probe_ty_var(vid) {
                         Ok(value) => bug!("resolved var in query: {goal:?} {value:?}"),
-                        Err(universe) => universe == self.universe(),
+                        Err(universe) => universe == self.infcx.universe(),
                     }
                 } else {
                     false
@@ -84,7 +458,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
                 if let ty::ConstKind::Infer(ty::InferConst::Var(vid)) = ct.kind() {
                     match self.infcx.probe_const_var(vid) {
                         Ok(value) => bug!("resolved var in query: {goal:?} {value:?}"),
-                        Err(universe) => universe == self.universe(),
+                        Err(universe) => universe == self.infcx.universe(),
                     }
                 } else {
                     false
@@ -93,37 +467,42 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         };
 
         // Guard against `<T as Trait<?0>>::Assoc = ?0>`.
-        struct ContainsTerm<'tcx> {
+        struct ContainsTerm<'a, 'tcx> {
             term: ty::Term<'tcx>,
+            infcx: &'a InferCtxt<'tcx>,
         }
-        impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ContainsTerm<'tcx> {
+        impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ContainsTerm<'_, 'tcx> {
             type BreakTy = ();
             fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
-                if t.needs_infer() {
-                    if ty::Term::from(t) == self.term {
-                        ControlFlow::Break(())
-                    } else {
-                        t.super_visit_with(self)
-                    }
+                if let Some(vid) = t.ty_vid()
+                    && let ty::TermKind::Ty(term) = self.term.unpack()
+                    && let Some(term_vid) = term.ty_vid()
+                    && self.infcx.root_var(vid) == self.infcx.root_var(term_vid)
+                {
+                    ControlFlow::Break(())
+                } else if t.has_non_region_infer() {
+                    t.super_visit_with(self)
                 } else {
                     ControlFlow::Continue(())
                 }
             }
 
             fn visit_const(&mut self, c: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
-                if c.needs_infer() {
-                    if ty::Term::from(c) == self.term {
-                        ControlFlow::Break(())
-                    } else {
-                        c.super_visit_with(self)
-                    }
+                if let ty::ConstKind::Infer(ty::InferConst::Var(vid)) = c.kind()
+                    && let ty::TermKind::Const(term) = self.term.unpack()
+                    && let ty::ConstKind::Infer(ty::InferConst::Var(term_vid)) = term.kind()
+                    && self.infcx.root_const_var(vid) == self.infcx.root_const_var(term_vid)
+                {
+                    ControlFlow::Break(())
+                } else if c.has_non_region_infer() {
+                    c.super_visit_with(self)
                 } else {
                     ControlFlow::Continue(())
                 }
             }
         }
 
-        let mut visitor = ContainsTerm { term: goal.predicate.term };
+        let mut visitor = ContainsTerm { infcx: self.infcx, term: goal.predicate.term };
 
         term_is_infer
             && goal.predicate.projection_ty.visit_with(&mut visitor).is_continue()
@@ -132,6 +511,49 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
 
     #[instrument(level = "debug", skip(self, param_env), ret)]
     pub(super) fn eq<T: ToTrace<'tcx>>(
+        &mut self,
+        param_env: ty::ParamEnv<'tcx>,
+        lhs: T,
+        rhs: T,
+    ) -> Result<(), NoSolution> {
+        self.infcx
+            .at(&ObligationCause::dummy(), param_env)
+            .eq(DefineOpaqueTypes::No, lhs, rhs)
+            .map(|InferOk { value: (), obligations }| {
+                self.add_goals(obligations.into_iter().map(|o| o.into()));
+            })
+            .map_err(|e| {
+                debug!(?e, "failed to equate");
+                NoSolution
+            })
+    }
+
+    #[instrument(level = "debug", skip(self, param_env), ret)]
+    pub(super) fn sub<T: ToTrace<'tcx>>(
+        &mut self,
+        param_env: ty::ParamEnv<'tcx>,
+        sub: T,
+        sup: T,
+    ) -> Result<(), NoSolution> {
+        self.infcx
+            .at(&ObligationCause::dummy(), param_env)
+            .sub(DefineOpaqueTypes::No, sub, sup)
+            .map(|InferOk { value: (), obligations }| {
+                self.add_goals(obligations.into_iter().map(|o| o.into()));
+            })
+            .map_err(|e| {
+                debug!(?e, "failed to subtype");
+                NoSolution
+            })
+    }
+
+    /// Equates two values returning the nested goals without adding them
+    /// to the nested goals of the `EvalCtxt`.
+    ///
+    /// If possible, try using `eq` instead which automatically handles nested
+    /// goals correctly.
+    #[instrument(level = "trace", skip(self, param_env), ret)]
+    pub(super) fn eq_and_get_goals<T: ToTrace<'tcx>>(
         &self,
         param_env: ty::ParamEnv<'tcx>,
         lhs: T,
@@ -139,7 +561,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ) -> Result<Vec<Goal<'tcx, ty::Predicate<'tcx>>>, NoSolution> {
         self.infcx
             .at(&ObligationCause::dummy(), param_env)
-            .eq(lhs, rhs)
+            .eq(DefineOpaqueTypes::No, lhs, rhs)
             .map(|InferOk { value: (), obligations }| {
                 obligations.into_iter().map(|o| o.into()).collect()
             })
@@ -178,7 +600,64 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         self.infcx.fresh_substs_for_item(DUMMY_SP, def_id)
     }
 
-    pub(super) fn universe(&self) -> ty::UniverseIndex {
-        self.infcx.universe()
+    pub(super) fn translate_substs(
+        &self,
+        param_env: ty::ParamEnv<'tcx>,
+        source_impl: DefId,
+        source_substs: ty::SubstsRef<'tcx>,
+        target_node: specialization_graph::Node,
+    ) -> ty::SubstsRef<'tcx> {
+        crate::traits::translate_substs(
+            self.infcx,
+            param_env,
+            source_impl,
+            source_substs,
+            target_node,
+        )
+    }
+
+    pub(super) fn register_ty_outlives(&self, ty: Ty<'tcx>, lt: ty::Region<'tcx>) {
+        self.infcx.register_region_obligation_with_cause(ty, lt, &ObligationCause::dummy());
+    }
+
+    pub(super) fn register_region_outlives(&self, a: ty::Region<'tcx>, b: ty::Region<'tcx>) {
+        // `b : a` ==> `a <= b`
+        // (inlined from `InferCtxt::region_outlives_predicate`)
+        self.infcx.sub_regions(
+            rustc_infer::infer::SubregionOrigin::RelateRegionParamBound(DUMMY_SP),
+            b,
+            a,
+        );
+    }
+
+    /// Computes the list of goals required for `arg` to be well-formed
+    pub(super) fn well_formed_goals(
+        &self,
+        param_env: ty::ParamEnv<'tcx>,
+        arg: ty::GenericArg<'tcx>,
+    ) -> Option<impl Iterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>> {
+        crate::traits::wf::unnormalized_obligations(self.infcx, param_env, arg)
+            .map(|obligations| obligations.into_iter().map(|obligation| obligation.into()))
+    }
+
+    pub(super) fn is_transmutable(
+        &self,
+        src_and_dst: rustc_transmute::Types<'tcx>,
+        scope: Ty<'tcx>,
+        assume: rustc_transmute::Assume,
+    ) -> Result<Certainty, NoSolution> {
+        // FIXME(transmutability): This really should be returning nested goals for `Answer::If*`
+        match rustc_transmute::TransmuteTypeEnv::new(self.infcx).is_transmutable(
+            ObligationCause::dummy(),
+            ty::Binder::dummy(src_and_dst),
+            scope,
+            assume,
+        ) {
+            rustc_transmute::Answer::Yes => Ok(Certainty::Yes),
+            rustc_transmute::Answer::No(_)
+            | rustc_transmute::Answer::IfTransmutable { .. }
+            | rustc_transmute::Answer::IfAll(_)
+            | rustc_transmute::Answer::IfAny(_) => Err(NoSolution),
+        }
     }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/canonical/mod.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
index 8c3be8da1..ada868705 100644
--- a/compiler/rustc_trait_selection/src/solve/canonical/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
@@ -8,22 +8,19 @@
 /// section of the [rustc-dev-guide][c].
 ///
 /// [c]: https://rustc-dev-guide.rust-lang.org/solve/canonicalization.html
-use self::canonicalize::{CanonicalizeMode, Canonicalizer};
 use super::{CanonicalGoal, Certainty, EvalCtxt, Goal};
-use super::{CanonicalResponse, ExternalConstraints, QueryResult, Response};
+use crate::solve::canonicalize::{CanonicalizeMode, Canonicalizer};
+use crate::solve::{CanonicalResponse, QueryResult, Response};
 use rustc_infer::infer::canonical::query_response::make_query_region_constraints;
 use rustc_infer::infer::canonical::CanonicalVarValues;
 use rustc_infer::infer::canonical::{CanonicalExt, QueryRegionConstraints};
-use rustc_infer::traits::query::NoSolution;
-use rustc_infer::traits::solve::ExternalConstraintsData;
-use rustc_infer::traits::ObligationCause;
+use rustc_middle::traits::query::NoSolution;
+use rustc_middle::traits::solve::{ExternalConstraints, ExternalConstraintsData};
 use rustc_middle::ty::{self, GenericArgKind};
 use rustc_span::DUMMY_SP;
 use std::iter;
 use std::ops::Deref;
 
-mod canonicalize;
-
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     /// Canonicalizes the goal remembering the original values
     /// for each bound variable.
@@ -45,10 +42,16 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ///
     /// - `var_values`: a map from bound variables in the canonical goal to
     ///   the values inferred while solving the instantiated goal.
-    /// - `external_constraints`: additional constraints which aren't expressable
+    /// - `external_constraints`: additional constraints which aren't expressible
     ///   using simple unification of inference variables.
     #[instrument(level = "debug", skip(self))]
-    pub(super) fn make_canonical_response(&self, certainty: Certainty) -> QueryResult<'tcx> {
+    pub(in crate::solve) fn evaluate_added_goals_and_make_canonical_response(
+        &mut self,
+        certainty: Certainty,
+    ) -> QueryResult<'tcx> {
+        let goals_certainty = self.try_evaluate_added_goals()?;
+        let certainty = certainty.unify_with(goals_certainty);
+
         let external_constraints = self.compute_external_query_constraints()?;
 
         let response = Response { var_values: self.var_values, external_constraints, certainty };
@@ -93,24 +96,24 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         param_env: ty::ParamEnv<'tcx>,
         original_values: Vec<ty::GenericArg<'tcx>>,
         response: CanonicalResponse<'tcx>,
-    ) -> Result<Certainty, NoSolution> {
+    ) -> Result<(Certainty, Vec<Goal<'tcx, ty::Predicate<'tcx>>>), NoSolution> {
         let substitution = self.compute_query_response_substitution(&original_values, &response);
 
         let Response { var_values, external_constraints, certainty } =
             response.substitute(self.tcx(), &substitution);
 
-        self.unify_query_var_values(param_env, &original_values, var_values)?;
+        let nested_goals = self.unify_query_var_values(param_env, &original_values, var_values)?;
 
         // FIXME: implement external constraints.
         let ExternalConstraintsData { region_constraints, opaque_types: _ } =
             external_constraints.deref();
         self.register_region_constraints(region_constraints);
 
-        Ok(certainty)
+        Ok((certainty, nested_goals))
     }
 
     /// This returns the substitutions to instantiate the bound variables of
-    /// the canonical reponse. This depends on the `original_values` for the
+    /// the canonical response. This depends on the `original_values` for the
     /// bound variables.
     fn compute_query_response_substitution(
         &self,
@@ -185,7 +188,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
                 } else {
                     // For placeholders which were already part of the input, we simply map this
                     // universal bound variable back the placeholder of the input.
-                    original_values[info.expect_anon_placeholder() as usize]
+                    original_values[info.expect_placeholder_index()]
                 }
             },
         ));
@@ -199,35 +202,22 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         param_env: ty::ParamEnv<'tcx>,
         original_values: &[ty::GenericArg<'tcx>],
         var_values: CanonicalVarValues<'tcx>,
-    ) -> Result<(), NoSolution> {
+    ) -> Result<Vec<Goal<'tcx, ty::Predicate<'tcx>>>, NoSolution> {
         assert_eq!(original_values.len(), var_values.len());
+
+        let mut nested_goals = vec![];
         for (&orig, response) in iter::zip(original_values, var_values.var_values) {
-            // This can fail due to the occurs check, see
-            // `tests/ui/typeck/lazy-norm/equating-projection-cyclically.rs` for an example
-            // where that can happen.
-            //
-            // FIXME: To deal with #105787 I also expect us to emit nested obligations here at
-            // some point. We can figure out how to deal with this once we actually have
-            // an ICE.
-            let nested_goals = self.eq(param_env, orig, response)?;
-            assert!(nested_goals.is_empty(), "{nested_goals:?}");
+            nested_goals.extend(self.eq_and_get_goals(param_env, orig, response)?);
         }
 
-        Ok(())
+        Ok(nested_goals)
     }
 
     fn register_region_constraints(&mut self, region_constraints: &QueryRegionConstraints<'tcx>) {
         for &(ty::OutlivesPredicate(lhs, rhs), _) in &region_constraints.outlives {
             match lhs.unpack() {
-                GenericArgKind::Lifetime(lhs) => self.infcx.region_outlives_predicate(
-                    &ObligationCause::dummy(),
-                    ty::Binder::dummy(ty::OutlivesPredicate(lhs, rhs)),
-                ),
-                GenericArgKind::Type(lhs) => self.infcx.register_region_obligation_with_cause(
-                    lhs,
-                    rhs,
-                    &ObligationCause::dummy(),
-                ),
+                GenericArgKind::Lifetime(lhs) => self.register_region_outlives(lhs, rhs),
+                GenericArgKind::Type(lhs) => self.register_ty_outlives(lhs, rhs),
                 GenericArgKind::Const(_) => bug!("const outlives: {lhs:?}: {rhs:?}"),
             }
         }
diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs
index a55b984fd..32bd10f0b 100644
--- a/compiler/rustc_trait_selection/src/solve/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs
@@ -1,6 +1,8 @@
 use std::mem;
 
 use rustc_infer::infer::InferCtxt;
+use rustc_infer::traits::solve::MaybeCause;
+use rustc_infer::traits::Obligation;
 use rustc_infer::traits::{
     query::NoSolution, FulfillmentError, FulfillmentErrorCode, MismatchedProjectionTypes,
     PredicateObligation, SelectionError, TraitEngine,
@@ -40,13 +42,31 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
         self.obligations.push(obligation);
     }
 
-    fn collect_remaining_errors(&mut self) -> Vec<FulfillmentError<'tcx>> {
+    fn collect_remaining_errors(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>> {
         self.obligations
             .drain(..)
-            .map(|obligation| FulfillmentError {
-                obligation: obligation.clone(),
-                code: FulfillmentErrorCode::CodeAmbiguity,
-                root_obligation: obligation,
+            .map(|obligation| {
+                let code =
+                    infcx.probe(|_| match infcx.evaluate_root_goal(obligation.clone().into()) {
+                        Ok((_, Certainty::Maybe(MaybeCause::Ambiguity), _)) => {
+                            FulfillmentErrorCode::CodeAmbiguity { overflow: false }
+                        }
+                        Ok((_, Certainty::Maybe(MaybeCause::Overflow), _)) => {
+                            FulfillmentErrorCode::CodeAmbiguity { overflow: true }
+                        }
+                        Ok((_, Certainty::Yes, _)) => {
+                            bug!("did not expect successful goal when collecting ambiguity errors")
+                        }
+                        Err(_) => {
+                            bug!("did not expect selection error when collecting ambiguity errors")
+                        }
+                    });
+
+                FulfillmentError {
+                    obligation: obligation.clone(),
+                    code,
+                    root_obligation: obligation,
+                }
             })
             .collect()
     }
@@ -61,7 +81,7 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
             let mut has_changed = false;
             for obligation in mem::take(&mut self.obligations) {
                 let goal = obligation.clone().into();
-                let (changed, certainty) = match infcx.evaluate_root_goal(goal) {
+                let (changed, certainty, nested_goals) = match infcx.evaluate_root_goal(goal) {
                     Ok(result) => result,
                     Err(NoSolution) => {
                         errors.push(FulfillmentError {
@@ -73,7 +93,7 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
                                         MismatchedProjectionTypes { err: TypeError::Mismatch },
                                     )
                                 }
-                                ty::PredicateKind::AliasEq(_, _) => {
+                                ty::PredicateKind::AliasRelate(_, _, _) => {
                                     FulfillmentErrorCode::CodeProjectionError(
                                         MismatchedProjectionTypes { err: TypeError::Mismatch },
                                     )
@@ -125,7 +145,16 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
                         continue;
                     }
                 };
-
+                // Push any nested goals that we get from unifying our canonical response
+                // with our obligation onto the fulfillment context.
+                self.obligations.extend(nested_goals.into_iter().map(|goal| {
+                    Obligation::new(
+                        infcx.tcx,
+                        obligation.cause.clone(),
+                        goal.param_env,
+                        goal.predicate,
+                    )
+                }));
                 has_changed |= changed;
                 match certainty {
                     Certainty::Yes => {}
@@ -149,6 +178,6 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
         &mut self,
         _: &InferCtxt<'tcx>,
     ) -> Vec<PredicateObligation<'tcx>> {
-        unimplemented!()
+        std::mem::take(&mut self.obligations)
     }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs
index 57b6a4527..19bcbd461 100644
--- a/compiler/rustc_trait_selection/src/solve/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/mod.rs
@@ -9,81 +9,45 @@
 //! FIXME(@lcnr): Write that section. If you read this before then ask me
 //! about it on zulip.
 
-// FIXME: Instead of using `infcx.canonicalize_query` we have to add a new routine which
-// preserves universes and creates a unique var (in the highest universe) for each
-// appearance of a region.
-
-// FIXME: uses of `infcx.at` need to enable deferred projection equality once that's implemented.
-
-use std::mem;
-
 use rustc_hir::def_id::DefId;
 use rustc_infer::infer::canonical::{Canonical, CanonicalVarValues};
-use rustc_infer::infer::{InferCtxt, InferOk, TyCtxtInferExt};
 use rustc_infer::traits::query::NoSolution;
-use rustc_infer::traits::Obligation;
-use rustc_middle::traits::solve::{ExternalConstraints, ExternalConstraintsData};
+use rustc_middle::traits::solve::{
+    CanonicalResponse, Certainty, ExternalConstraintsData, Goal, QueryResult, Response,
+};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::ty::{
-    CoercePredicate, RegionOutlivesPredicate, SubtypePredicate, ToPredicate, TypeOutlivesPredicate,
+    CoercePredicate, RegionOutlivesPredicate, SubtypePredicate, TypeOutlivesPredicate,
 };
-use rustc_span::DUMMY_SP;
-
-use crate::solve::search_graph::OverflowHandler;
-use crate::traits::ObligationCause;
 
 mod assembly;
-mod canonical;
+mod canonicalize;
 mod eval_ctxt;
 mod fulfill;
 mod project_goals;
 mod search_graph;
 mod trait_goals;
 
-pub use eval_ctxt::EvalCtxt;
+pub use eval_ctxt::{EvalCtxt, InferCtxtEvalExt};
 pub use fulfill::FulfillmentCtxt;
 
-/// A goal is a statement, i.e. `predicate`, we want to prove
-/// given some assumptions, i.e. `param_env`.
-///
-/// Most of the time the `param_env` contains the `where`-bounds of the function
-/// we're currently typechecking while the `predicate` is some trait bound.
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
-pub struct Goal<'tcx, P> {
-    param_env: ty::ParamEnv<'tcx>,
-    predicate: P,
-}
-
-impl<'tcx, P> Goal<'tcx, P> {
-    pub fn new(
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-        predicate: impl ToPredicate<'tcx, P>,
-    ) -> Goal<'tcx, P> {
-        Goal { param_env, predicate: predicate.to_predicate(tcx) }
-    }
-
-    /// Updates the goal to one with a different `predicate` but the same `param_env`.
-    fn with<Q>(self, tcx: TyCtxt<'tcx>, predicate: impl ToPredicate<'tcx, Q>) -> Goal<'tcx, Q> {
-        Goal { param_env: self.param_env, predicate: predicate.to_predicate(tcx) }
-    }
-}
-
-impl<'tcx, P> From<Obligation<'tcx, P>> for Goal<'tcx, P> {
-    fn from(obligation: Obligation<'tcx, P>) -> Goal<'tcx, P> {
-        Goal { param_env: obligation.param_env, predicate: obligation.predicate }
-    }
-}
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
-pub struct Response<'tcx> {
-    pub var_values: CanonicalVarValues<'tcx>,
-    /// Additional constraints returned by this query.
-    pub external_constraints: ExternalConstraints<'tcx>,
-    pub certainty: Certainty,
+#[derive(Debug, Clone, Copy)]
+enum SolverMode {
+    /// Ordinary trait solving, using everywhere except for coherence.
+    Normal,
+    /// Trait solving during coherence. There are a few notable differences
+    /// between coherence and ordinary trait solving.
+    ///
+    /// Most importantly, trait solving during coherence must not be incomplete,
+    /// i.e. return `Err(NoSolution)` for goals for which a solution exists.
+    /// This means that we must not make any guesses or arbitrary choices.
+    Coherence,
 }
 
 trait CanonicalResponseExt {
     fn has_no_inference_or_external_constraints(&self) -> bool;
+
+    fn has_only_region_constraints(&self) -> bool;
 }
 
 impl<'tcx> CanonicalResponseExt for Canonical<'tcx, Response<'tcx>> {
@@ -92,242 +56,35 @@ impl<'tcx> CanonicalResponseExt for Canonical<'tcx, Response<'tcx>> {
             && self.value.var_values.is_identity()
             && self.value.external_constraints.opaque_types.is_empty()
     }
-}
 
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
-pub enum Certainty {
-    Yes,
-    Maybe(MaybeCause),
-}
-
-impl Certainty {
-    pub const AMBIGUOUS: Certainty = Certainty::Maybe(MaybeCause::Ambiguity);
-
-    /// When proving multiple goals using **AND**, e.g. nested obligations for an impl,
-    /// use this function to unify the certainty of these goals
-    pub fn unify_and(self, other: Certainty) -> Certainty {
-        match (self, other) {
-            (Certainty::Yes, Certainty::Yes) => Certainty::Yes,
-            (Certainty::Yes, Certainty::Maybe(_)) => other,
-            (Certainty::Maybe(_), Certainty::Yes) => self,
-            (Certainty::Maybe(MaybeCause::Overflow), Certainty::Maybe(MaybeCause::Overflow)) => {
-                Certainty::Maybe(MaybeCause::Overflow)
-            }
-            // If at least one of the goals is ambiguous, hide the overflow as the ambiguous goal
-            // may still result in failure.
-            (Certainty::Maybe(MaybeCause::Ambiguity), Certainty::Maybe(_))
-            | (Certainty::Maybe(_), Certainty::Maybe(MaybeCause::Ambiguity)) => {
-                Certainty::Maybe(MaybeCause::Ambiguity)
-            }
-        }
-    }
-}
-
-/// Why we failed to evaluate a goal.
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
-pub enum MaybeCause {
-    /// We failed due to ambiguity. This ambiguity can either
-    /// be a true ambiguity, i.e. there are multiple different answers,
-    /// or we hit a case where we just don't bother, e.g. `?x: Trait` goals.
-    Ambiguity,
-    /// We gave up due to an overflow, most often by hitting the recursion limit.
-    Overflow,
-}
-
-type CanonicalGoal<'tcx, T = ty::Predicate<'tcx>> = Canonical<'tcx, Goal<'tcx, T>>;
-type CanonicalResponse<'tcx> = Canonical<'tcx, Response<'tcx>>;
-/// The result of evaluating a canonical query.
-///
-/// FIXME: We use a different type than the existing canonical queries. This is because
-/// we need to add a `Certainty` for `overflow` and may want to restructure this code without
-/// having to worry about changes to currently used code. Once we've made progress on this
-/// solver, merge the two responses again.
-pub type QueryResult<'tcx> = Result<CanonicalResponse<'tcx>, NoSolution>;
-
-pub trait InferCtxtEvalExt<'tcx> {
-    /// Evaluates a goal from **outside** of the trait solver.
-    ///
-    /// Using this while inside of the solver is wrong as it uses a new
-    /// search graph which would break cycle detection.
-    fn evaluate_root_goal(
-        &self,
-        goal: Goal<'tcx, ty::Predicate<'tcx>>,
-    ) -> Result<(bool, Certainty), NoSolution>;
-}
-
-impl<'tcx> InferCtxtEvalExt<'tcx> for InferCtxt<'tcx> {
-    fn evaluate_root_goal(
-        &self,
-        goal: Goal<'tcx, ty::Predicate<'tcx>>,
-    ) -> Result<(bool, Certainty), NoSolution> {
-        let mut search_graph = search_graph::SearchGraph::new(self.tcx);
-
-        let result = EvalCtxt {
-            search_graph: &mut search_graph,
-            infcx: self,
-            // Only relevant when canonicalizing the response.
-            max_input_universe: ty::UniverseIndex::ROOT,
-            var_values: CanonicalVarValues::dummy(),
-            in_projection_eq_hack: false,
-        }
-        .evaluate_goal(goal);
-
-        assert!(search_graph.is_empty());
-        result
+    fn has_only_region_constraints(&self) -> bool {
+        self.value.var_values.is_identity_modulo_regions()
+            && self.value.external_constraints.opaque_types.is_empty()
     }
 }
 
 impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
-    /// The entry point of the solver.
-    ///
-    /// This function deals with (coinductive) cycles, overflow, and caching
-    /// and then calls [`EvalCtxt::compute_goal`] which contains the actual
-    /// logic of the solver.
-    ///
-    /// Instead of calling this function directly, use either [EvalCtxt::evaluate_goal]
-    /// if you're inside of the solver or [InferCtxtEvalExt::evaluate_root_goal] if you're
-    /// outside of it.
-    #[instrument(level = "debug", skip(tcx, search_graph), ret)]
-    fn evaluate_canonical_goal(
-        tcx: TyCtxt<'tcx>,
-        search_graph: &'a mut search_graph::SearchGraph<'tcx>,
-        canonical_goal: CanonicalGoal<'tcx>,
-    ) -> QueryResult<'tcx> {
-        // Deal with overflow, caching, and coinduction.
-        //
-        // The actual solver logic happens in `ecx.compute_goal`.
-        search_graph.with_new_goal(tcx, canonical_goal, |search_graph| {
-            let (ref infcx, goal, var_values) =
-                tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical_goal);
-            let mut ecx = EvalCtxt {
-                infcx,
-                var_values,
-                max_input_universe: canonical_goal.max_universe,
-                search_graph,
-                in_projection_eq_hack: false,
-            };
-            ecx.compute_goal(goal)
-        })
-    }
-
-    /// Recursively evaluates `goal`, returning whether any inference vars have
-    /// been constrained and the certainty of the result.
-    fn evaluate_goal(
-        &mut self,
-        goal: Goal<'tcx, ty::Predicate<'tcx>>,
-    ) -> Result<(bool, Certainty), NoSolution> {
-        let (orig_values, canonical_goal) = self.canonicalize_goal(goal);
-        let canonical_response =
-            EvalCtxt::evaluate_canonical_goal(self.tcx(), self.search_graph, canonical_goal)?;
-
-        let has_changed = !canonical_response.value.var_values.is_identity();
-        let certainty = self.instantiate_and_apply_query_response(
-            goal.param_env,
-            orig_values,
-            canonical_response,
-        )?;
-
-        // Check that rerunning this query with its inference constraints applied
-        // doesn't result in new inference constraints and has the same result.
-        //
-        // If we have projection goals like `<T as Trait>::Assoc == u32` we recursively
-        // call `exists<U> <T as Trait>::Assoc == U` to enable better caching. This goal
-        // could constrain `U` to `u32` which would cause this check to result in a
-        // solver cycle.
-        if cfg!(debug_assertions)
-            && has_changed
-            && !self.in_projection_eq_hack
-            && !self.search_graph.in_cycle()
-        {
-            let (_orig_values, canonical_goal) = self.canonicalize_goal(goal);
-            let canonical_response =
-                EvalCtxt::evaluate_canonical_goal(self.tcx(), self.search_graph, canonical_goal)?;
-            if !canonical_response.value.var_values.is_identity() {
-                bug!("unstable result: {goal:?} {canonical_goal:?} {canonical_response:?}");
-            }
-            assert_eq!(certainty, canonical_response.value.certainty);
-        }
-
-        Ok((has_changed, certainty))
-    }
-
-    fn compute_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) -> QueryResult<'tcx> {
-        let Goal { param_env, predicate } = goal;
-        let kind = predicate.kind();
-        if let Some(kind) = kind.no_bound_vars() {
-            match kind {
-                ty::PredicateKind::Clause(ty::Clause::Trait(predicate)) => {
-                    self.compute_trait_goal(Goal { param_env, predicate })
-                }
-                ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) => {
-                    self.compute_projection_goal(Goal { param_env, predicate })
-                }
-                ty::PredicateKind::Clause(ty::Clause::TypeOutlives(predicate)) => {
-                    self.compute_type_outlives_goal(Goal { param_env, predicate })
-                }
-                ty::PredicateKind::Clause(ty::Clause::RegionOutlives(predicate)) => {
-                    self.compute_region_outlives_goal(Goal { param_env, predicate })
-                }
-                ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
-                    self.compute_const_arg_has_type_goal(Goal { param_env, predicate: (ct, ty) })
-                }
-                ty::PredicateKind::Subtype(predicate) => {
-                    self.compute_subtype_goal(Goal { param_env, predicate })
-                }
-                ty::PredicateKind::Coerce(predicate) => {
-                    self.compute_coerce_goal(Goal { param_env, predicate })
-                }
-                ty::PredicateKind::ClosureKind(def_id, substs, kind) => self
-                    .compute_closure_kind_goal(Goal {
-                        param_env,
-                        predicate: (def_id, substs, kind),
-                    }),
-                ty::PredicateKind::ObjectSafe(trait_def_id) => {
-                    self.compute_object_safe_goal(trait_def_id)
-                }
-                ty::PredicateKind::WellFormed(arg) => {
-                    self.compute_well_formed_goal(Goal { param_env, predicate: arg })
-                }
-                ty::PredicateKind::Ambiguous => self.make_canonical_response(Certainty::AMBIGUOUS),
-                // FIXME: implement these predicates :)
-                ty::PredicateKind::ConstEvaluatable(_) | ty::PredicateKind::ConstEquate(_, _) => {
-                    self.make_canonical_response(Certainty::Yes)
-                }
-                ty::PredicateKind::TypeWellFormedFromEnv(..) => {
-                    bug!("TypeWellFormedFromEnv is only used for Chalk")
-                }
-                ty::PredicateKind::AliasEq(lhs, rhs) => {
-                    self.compute_alias_eq_goal(Goal { param_env, predicate: (lhs, rhs) })
-                }
-            }
-        } else {
-            let kind = self.infcx.instantiate_binder_with_placeholders(kind);
-            let goal = goal.with(self.tcx(), ty::Binder::dummy(kind));
-            let (_, certainty) = self.evaluate_goal(goal)?;
-            self.make_canonical_response(certainty)
-        }
-    }
-
+    #[instrument(level = "debug", skip(self))]
     fn compute_type_outlives_goal(
         &mut self,
         goal: Goal<'tcx, TypeOutlivesPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
         let ty::OutlivesPredicate(ty, lt) = goal.predicate;
-        self.infcx.register_region_obligation_with_cause(ty, lt, &ObligationCause::dummy());
-        self.make_canonical_response(Certainty::Yes)
+        self.register_ty_outlives(ty, lt);
+        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn compute_region_outlives_goal(
         &mut self,
         goal: Goal<'tcx, RegionOutlivesPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
-        self.infcx.region_outlives_predicate(
-            &ObligationCause::dummy(),
-            ty::Binder::dummy(goal.predicate),
-        );
-        self.make_canonical_response(Certainty::Yes)
+        let ty::OutlivesPredicate(a, b) = goal.predicate;
+        self.register_region_outlives(a, b);
+        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn compute_coerce_goal(
         &mut self,
         goal: Goal<'tcx, CoercePredicate<'tcx>>,
@@ -342,25 +99,20 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
         })
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn compute_subtype_goal(
         &mut self,
         goal: Goal<'tcx, SubtypePredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
         if goal.predicate.a.is_ty_var() && goal.predicate.b.is_ty_var() {
-            // FIXME: Do we want to register a subtype relation between these vars?
-            // That won't actually reflect in the query response, so it seems moot.
-            self.make_canonical_response(Certainty::AMBIGUOUS)
+            self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
         } else {
-            let InferOk { value: (), obligations } = self
-                .infcx
-                .at(&ObligationCause::dummy(), goal.param_env)
-                .sub(goal.predicate.a, goal.predicate.b)?;
-            self.evaluate_all_and_make_canonical_response(
-                obligations.into_iter().map(|pred| pred.into()).collect(),
-            )
+            self.sub(goal.param_env, goal.predicate.a, goal.predicate.b)?;
+            self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         }
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn compute_closure_kind_goal(
         &mut self,
         goal: Goal<'tcx, (DefId, ty::SubstsRef<'tcx>, ty::ClosureKind)>,
@@ -369,92 +121,154 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
         let found_kind = substs.as_closure().kind_ty().to_opt_closure_kind();
 
         let Some(found_kind) = found_kind else {
-            return self.make_canonical_response(Certainty::AMBIGUOUS);
+            return self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
         };
         if found_kind.extends(expected_kind) {
-            self.make_canonical_response(Certainty::Yes)
+            self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         } else {
             Err(NoSolution)
         }
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn compute_object_safe_goal(&mut self, trait_def_id: DefId) -> QueryResult<'tcx> {
         if self.tcx().check_is_object_safe(trait_def_id) {
-            self.make_canonical_response(Certainty::Yes)
+            self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         } else {
             Err(NoSolution)
         }
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn compute_well_formed_goal(
         &mut self,
         goal: Goal<'tcx, ty::GenericArg<'tcx>>,
     ) -> QueryResult<'tcx> {
-        match crate::traits::wf::unnormalized_obligations(
-            self.infcx,
-            goal.param_env,
-            goal.predicate,
-        ) {
-            Some(obligations) => self.evaluate_all_and_make_canonical_response(
-                obligations.into_iter().map(|o| o.into()).collect(),
-            ),
-            None => self.make_canonical_response(Certainty::AMBIGUOUS),
+        match self.well_formed_goals(goal.param_env, goal.predicate) {
+            Some(goals) => {
+                self.add_goals(goals);
+                self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+            }
+            None => self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS),
         }
     }
 
     #[instrument(level = "debug", skip(self), ret)]
-    fn compute_alias_eq_goal(
+    fn compute_alias_relate_goal(
         &mut self,
-        goal: Goal<'tcx, (ty::Term<'tcx>, ty::Term<'tcx>)>,
+        goal: Goal<'tcx, (ty::Term<'tcx>, ty::Term<'tcx>, ty::AliasRelationDirection)>,
     ) -> QueryResult<'tcx> {
         let tcx = self.tcx();
+        // We may need to invert the alias relation direction if dealing an alias on the RHS.
+        #[derive(Debug)]
+        enum Invert {
+            No,
+            Yes,
+        }
+        let evaluate_normalizes_to =
+            |ecx: &mut EvalCtxt<'_, 'tcx>, alias, other, direction, invert| {
+                let span = tracing::span!(
+                    tracing::Level::DEBUG,
+                    "compute_alias_relate_goal(evaluate_normalizes_to)",
+                    ?alias,
+                    ?other,
+                    ?direction,
+                    ?invert
+                );
+                let _enter = span.enter();
+                let result = ecx.probe(|ecx| {
+                    let other = match direction {
+                        // This is purely an optimization.
+                        ty::AliasRelationDirection::Equate => other,
+
+                        ty::AliasRelationDirection::Subtype => {
+                            let fresh = ecx.next_term_infer_of_kind(other);
+                            let (sub, sup) = match invert {
+                                Invert::No => (fresh, other),
+                                Invert::Yes => (other, fresh),
+                            };
+                            ecx.sub(goal.param_env, sub, sup)?;
+                            fresh
+                        }
+                    };
+                    ecx.add_goal(goal.with(
+                        tcx,
+                        ty::Binder::dummy(ty::ProjectionPredicate {
+                            projection_ty: alias,
+                            term: other,
+                        }),
+                    ));
+                    ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+                });
+                debug!(?result);
+                result
+            };
 
-        let evaluate_normalizes_to = |ecx: &mut EvalCtxt<'_, 'tcx>, alias, other| {
-            debug!("evaluate_normalizes_to(alias={:?}, other={:?})", alias, other);
-            let r = ecx.probe(|ecx| {
-                let (_, certainty) = ecx.evaluate_goal(goal.with(
-                    tcx,
-                    ty::Binder::dummy(ty::ProjectionPredicate {
-                        projection_ty: alias,
-                        term: other,
-                    }),
-                ))?;
-                ecx.make_canonical_response(certainty)
-            });
-            debug!("evaluate_normalizes_to(..) -> {:?}", r);
-            r
-        };
+        let (lhs, rhs, direction) = goal.predicate;
 
-        if goal.predicate.0.is_infer() || goal.predicate.1.is_infer() {
+        if lhs.is_infer() || rhs.is_infer() {
             bug!(
-                "`AliasEq` goal with an infer var on lhs or rhs which should have been instantiated"
+                "`AliasRelate` goal with an infer var on lhs or rhs which should have been instantiated"
             );
         }
 
-        match (
-            goal.predicate.0.to_alias_term_no_opaque(tcx),
-            goal.predicate.1.to_alias_term_no_opaque(tcx),
-        ) {
-            (None, None) => bug!("`AliasEq` goal without an alias on either lhs or rhs"),
-            (Some(alias), None) => evaluate_normalizes_to(self, alias, goal.predicate.1),
-            (None, Some(alias)) => evaluate_normalizes_to(self, alias, goal.predicate.0),
-            (Some(alias_lhs), Some(alias_rhs)) => {
-                debug!("compute_alias_eq_goal: both sides are aliases");
+        match (lhs.to_projection_term(tcx), rhs.to_projection_term(tcx)) {
+            (None, None) => bug!("`AliasRelate` goal without an alias on either lhs or rhs"),
 
-                let mut candidates = Vec::with_capacity(3);
+            // RHS is not a projection, only way this is true is if LHS normalizes-to RHS
+            (Some(alias_lhs), None) => {
+                evaluate_normalizes_to(self, alias_lhs, rhs, direction, Invert::No)
+            }
+
+            // LHS is not a projection, only way this is true is if RHS normalizes-to LHS
+            (None, Some(alias_rhs)) => {
+                evaluate_normalizes_to(self, alias_rhs, lhs, direction, Invert::Yes)
+            }
 
-                // Evaluate all 3 potential candidates for the alias' being equal
-                candidates.push(evaluate_normalizes_to(self, alias_lhs, goal.predicate.1));
-                candidates.push(evaluate_normalizes_to(self, alias_rhs, goal.predicate.0));
-                candidates.push(self.probe(|this| {
-                    debug!("compute_alias_eq_goal: alias defids are equal, equating substs");
-                    let nested_goals = this.eq(goal.param_env, alias_lhs, alias_rhs)?;
-                    this.evaluate_all_and_make_canonical_response(nested_goals)
-                }));
+            (Some(alias_lhs), Some(alias_rhs)) => {
+                debug!("both sides are aliases");
+
+                let mut candidates = Vec::new();
+                // LHS normalizes-to RHS
+                candidates.extend(
+                    evaluate_normalizes_to(self, alias_lhs, rhs, direction, Invert::No).ok(),
+                );
+                // RHS normalizes-to RHS
+                candidates.extend(
+                    evaluate_normalizes_to(self, alias_rhs, lhs, direction, Invert::Yes).ok(),
+                );
+                // Relate via substs
+                candidates.extend(
+                    self.probe(|ecx| {
+                        let span = tracing::span!(
+                            tracing::Level::DEBUG,
+                            "compute_alias_relate_goal(relate_via_substs)",
+                            ?alias_lhs,
+                            ?alias_rhs,
+                            ?direction
+                        );
+                        let _enter = span.enter();
+
+                        match direction {
+                            ty::AliasRelationDirection::Equate => {
+                                ecx.eq(goal.param_env, alias_lhs, alias_rhs)?;
+                            }
+                            ty::AliasRelationDirection::Subtype => {
+                                ecx.sub(goal.param_env, alias_lhs, alias_rhs)?;
+                            }
+                        }
 
+                        ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+                    })
+                    .ok(),
+                );
                 debug!(?candidates);
 
-                self.try_merge_responses(candidates.into_iter())
+                if let Some(merged) = self.try_merge_responses(&candidates) {
+                    Ok(merged)
+                } else {
+                    self.flounder(&candidates)
+                }
             }
         }
     }
@@ -465,99 +279,78 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
         goal: Goal<'tcx, (ty::Const<'tcx>, Ty<'tcx>)>,
     ) -> QueryResult<'tcx> {
         let (ct, ty) = goal.predicate;
-        let nested_goals = self.eq(goal.param_env, ct.ty(), ty)?;
-        self.evaluate_all_and_make_canonical_response(nested_goals)
+        self.eq(goal.param_env, ct.ty(), ty)?;
+        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 }
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
-    // Recursively evaluates a list of goals to completion, returning the certainty
-    // of all of the goals.
-    fn evaluate_all(
-        &mut self,
-        mut goals: Vec<Goal<'tcx, ty::Predicate<'tcx>>>,
-    ) -> Result<Certainty, NoSolution> {
-        let mut new_goals = Vec::new();
-        self.repeat_while_none(
-            |_| Ok(Certainty::Maybe(MaybeCause::Overflow)),
-            |this| {
-                let mut has_changed = Err(Certainty::Yes);
-                for goal in goals.drain(..) {
-                    let (changed, certainty) = match this.evaluate_goal(goal) {
-                        Ok(result) => result,
-                        Err(NoSolution) => return Some(Err(NoSolution)),
-                    };
-
-                    if changed {
-                        has_changed = Ok(());
-                    }
-
-                    match certainty {
-                        Certainty::Yes => {}
-                        Certainty::Maybe(_) => {
-                            new_goals.push(goal);
-                            has_changed = has_changed.map_err(|c| c.unify_and(certainty));
-                        }
-                    }
-                }
+    #[instrument(level = "debug", skip(self))]
+    fn set_normalizes_to_hack_goal(&mut self, goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>) {
+        assert!(
+            self.nested_goals.normalizes_to_hack_goal.is_none(),
+            "attempted to set the projection eq hack goal when one already exists"
+        );
+        self.nested_goals.normalizes_to_hack_goal = Some(goal);
+    }
 
-                match has_changed {
-                    Ok(()) => {
-                        mem::swap(&mut new_goals, &mut goals);
-                        None
-                    }
-                    Err(certainty) => Some(Ok(certainty)),
-                }
-            },
-        )
+    #[instrument(level = "debug", skip(self))]
+    fn add_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) {
+        self.nested_goals.goals.push(goal);
     }
 
-    // Recursively evaluates a list of goals to completion, making a query response.
-    //
-    // This is just a convenient way of calling [`EvalCtxt::evaluate_all`],
-    // then [`EvalCtxt::make_canonical_response`].
-    fn evaluate_all_and_make_canonical_response(
-        &mut self,
-        goals: Vec<Goal<'tcx, ty::Predicate<'tcx>>>,
-    ) -> QueryResult<'tcx> {
-        self.evaluate_all(goals).and_then(|certainty| self.make_canonical_response(certainty))
+    #[instrument(level = "debug", skip(self, goals))]
+    fn add_goals(&mut self, goals: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>) {
+        let current_len = self.nested_goals.goals.len();
+        self.nested_goals.goals.extend(goals);
+        debug!("added_goals={:?}", &self.nested_goals.goals[current_len..]);
     }
 
+    /// Try to merge multiple possible ways to prove a goal, if that is not possible returns `None`.
+    ///
+    /// In this case we tend to flounder and return ambiguity by calling `[EvalCtxt::flounder]`.
+    #[instrument(level = "debug", skip(self), ret)]
     fn try_merge_responses(
         &mut self,
-        responses: impl Iterator<Item = QueryResult<'tcx>>,
-    ) -> QueryResult<'tcx> {
-        let candidates = responses.into_iter().flatten().collect::<Box<[_]>>();
-
-        if candidates.is_empty() {
-            return Err(NoSolution);
+        responses: &[CanonicalResponse<'tcx>],
+    ) -> Option<CanonicalResponse<'tcx>> {
+        if responses.is_empty() {
+            return None;
         }
 
-        // FIXME(-Ztreat-solver=next): We should instead try to find a `Certainty::Yes` response with
+        // FIXME(-Ztrait-solver=next): We should instead try to find a `Certainty::Yes` response with
         // a subset of the constraints that all the other responses have.
-        let one = candidates[0];
-        if candidates[1..].iter().all(|resp| resp == &one) {
-            return Ok(one);
+        let one = responses[0];
+        if responses[1..].iter().all(|&resp| resp == one) {
+            return Some(one);
         }
 
-        if let Some(response) = candidates.iter().find(|response| {
-            response.value.certainty == Certainty::Yes
-                && response.has_no_inference_or_external_constraints()
-        }) {
-            return Ok(*response);
-        }
+        responses
+            .iter()
+            .find(|response| {
+                response.value.certainty == Certainty::Yes
+                    && response.has_no_inference_or_external_constraints()
+            })
+            .copied()
+    }
 
-        let certainty = candidates.iter().fold(Certainty::AMBIGUOUS, |certainty, response| {
-            certainty.unify_and(response.value.certainty)
+    /// If we fail to merge responses we flounder and return overflow or ambiguity.
+    #[instrument(level = "debug", skip(self), ret)]
+    fn flounder(&mut self, responses: &[CanonicalResponse<'tcx>]) -> QueryResult<'tcx> {
+        if responses.is_empty() {
+            return Err(NoSolution);
+        }
+        let certainty = responses.iter().fold(Certainty::AMBIGUOUS, |certainty, response| {
+            certainty.unify_with(response.value.certainty)
         });
-        // FIXME(-Ztrait-solver=next): We should take the intersection of the constraints on all the
-        // responses and use that for the constraints of this ambiguous response.
-        let response = self.make_canonical_response(certainty);
-        if let Ok(response) = &response {
+
+        let response = self.evaluate_added_goals_and_make_canonical_response(certainty);
+        if let Ok(response) = response {
             assert!(response.has_no_inference_or_external_constraints());
+            Ok(response)
+        } else {
+            bug!("failed to make floundered response: {responses:?}");
         }
-
-        response
     }
 }
 
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals.rs b/compiler/rustc_trait_selection/src/solve/project_goals.rs
index 33c66d072..14cb43b89 100644
--- a/compiler/rustc_trait_selection/src/solve/project_goals.rs
+++ b/compiler/rustc_trait_selection/src/solve/project_goals.rs
@@ -1,24 +1,23 @@
-use crate::traits::{specialization_graph, translate_substs};
+use crate::traits::specialization_graph;
 
-use super::assembly;
-use super::trait_goals::structural_traits;
-use super::{Certainty, EvalCtxt, Goal, QueryResult};
+use super::assembly::{self, structural_traits};
+use super::EvalCtxt;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::LangItem;
-use rustc_infer::infer::InferCtxt;
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::specialization_graph::LeafDef;
 use rustc_infer::traits::Reveal;
+use rustc_middle::traits::solve::{CanonicalResponse, Certainty, Goal, QueryResult};
 use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
 use rustc_middle::ty::ProjectionPredicate;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::ty::{ToPredicate, TypeVisitableExt};
 use rustc_span::{sym, DUMMY_SP};
-use std::iter;
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
+    #[instrument(level = "debug", skip(self), ret)]
     pub(super) fn compute_projection_goal(
         &mut self,
         goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
@@ -32,57 +31,12 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         // projection cache in the solver.
         if self.term_is_fully_unconstrained(goal) {
             let candidates = self.assemble_and_evaluate_candidates(goal);
-            self.merge_candidates_and_discard_reservation_impls(candidates)
+            self.merge_candidates(candidates)
         } else {
-            let predicate = goal.predicate;
-            let unconstrained_rhs = match predicate.term.unpack() {
-                ty::TermKind::Ty(_) => self.next_ty_infer().into(),
-                ty::TermKind::Const(ct) => self.next_const_infer(ct.ty()).into(),
-            };
-            let unconstrained_predicate = ty::Clause::Projection(ProjectionPredicate {
-                projection_ty: goal.predicate.projection_ty,
-                term: unconstrained_rhs,
-            });
-            let (_has_changed, normalize_certainty) = self.in_projection_eq_hack(|this| {
-                this.evaluate_goal(goal.with(this.tcx(), unconstrained_predicate))
-            })?;
-
-            let nested_eq_goals = self.eq(goal.param_env, unconstrained_rhs, predicate.term)?;
-            let eval_certainty = self.evaluate_all(nested_eq_goals)?;
-            self.make_canonical_response(normalize_certainty.unify_and(eval_certainty))
+            self.set_normalizes_to_hack_goal(goal);
+            self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         }
     }
-
-    /// This sets a flag used by a debug assert in [`EvalCtxt::evaluate_goal`],
-    /// see the comment in that method for more details.
-    fn in_projection_eq_hack<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
-        self.in_projection_eq_hack = true;
-        let result = f(self);
-        self.in_projection_eq_hack = false;
-        result
-    }
-
-    /// After normalizing the projection to `normalized_alias` with the given
-    /// `normalization_certainty`, constrain the inference variable `term` to it
-    /// and return a query response.
-    fn eq_term_and_make_canonical_response(
-        &mut self,
-        goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
-        normalization_certainty: Certainty,
-        normalized_alias: impl Into<ty::Term<'tcx>>,
-    ) -> QueryResult<'tcx> {
-        // The term of our goal should be fully unconstrained, so this should never fail.
-        //
-        // It can however be ambiguous when the `normalized_alias` contains a projection.
-        let nested_goals = self
-            .eq(goal.param_env, goal.predicate.term, normalized_alias.into())
-            .expect("failed to unify with unconstrained term");
-
-        let unify_certainty =
-            self.evaluate_all(nested_goals).expect("failed to unify with unconstrained term");
-
-        self.make_canonical_response(normalization_certainty.unify_and(unify_certainty))
-    }
 }
 
 impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
@@ -90,6 +44,10 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         self.self_ty()
     }
 
+    fn trait_ref(self, tcx: TyCtxt<'tcx>) -> ty::TraitRef<'tcx> {
+        self.projection_ty.trait_ref(tcx)
+    }
+
     fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
         self.with_self_ty(tcx, self_ty)
     }
@@ -110,19 +68,14 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
             ecx.probe(|ecx| {
                 let assumption_projection_pred =
                     ecx.instantiate_binder_with_infer(poly_projection_pred);
-                let mut nested_goals = ecx.eq(
+                ecx.eq(
                     goal.param_env,
                     goal.predicate.projection_ty,
                     assumption_projection_pred.projection_ty,
                 )?;
-                nested_goals.extend(requirements);
-                let subst_certainty = ecx.evaluate_all(nested_goals)?;
-
-                ecx.eq_term_and_make_canonical_response(
-                    goal,
-                    subst_certainty,
-                    assumption_projection_pred.term,
-                )
+                ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)?;
+                ecx.add_goals(requirements);
+                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             })
         } else {
             Err(NoSolution)
@@ -138,21 +91,22 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
             && poly_projection_pred.projection_def_id() == goal.predicate.def_id()
         {
             ecx.probe(|ecx| {
+                let tcx = ecx.tcx();
+
                 let assumption_projection_pred =
                     ecx.instantiate_binder_with_infer(poly_projection_pred);
-                let mut nested_goals = ecx.eq(
+                ecx.eq(
                     goal.param_env,
                     goal.predicate.projection_ty,
                     assumption_projection_pred.projection_ty,
                 )?;
 
-                let tcx = ecx.tcx();
                 let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else {
                     bug!("expected object type in `consider_object_bound_candidate`");
                 };
-                nested_goals.extend(
+                ecx.add_goals(
                     structural_traits::predicates_for_object_candidate(
-                        ecx,
+                        &ecx,
                         goal.param_env,
                         goal.predicate.projection_ty.trait_ref(tcx),
                         bounds,
@@ -160,14 +114,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                     .into_iter()
                     .map(|pred| goal.with(tcx, pred)),
                 );
-
-                let subst_certainty = ecx.evaluate_all(nested_goals)?;
-
-                ecx.eq_term_and_make_canonical_response(
-                    goal,
-                    subst_certainty,
-                    assumption_projection_pred.term,
-                )
+                ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)?;
+                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             })
         } else {
             Err(NoSolution)
@@ -183,10 +131,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
 
         let goal_trait_ref = goal.predicate.projection_ty.trait_ref(tcx);
         let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap();
-        let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsPlaceholder };
-        if iter::zip(goal_trait_ref.substs, impl_trait_ref.skip_binder().substs)
-            .any(|(goal, imp)| !drcx.generic_args_may_unify(goal, imp))
-        {
+        let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::ForLookup };
+        if !drcx.substs_refs_may_unify(goal_trait_ref.substs, impl_trait_ref.skip_binder().substs) {
             return Err(NoSolution);
         }
 
@@ -194,28 +140,27 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
             let impl_substs = ecx.fresh_substs_for_item(impl_def_id);
             let impl_trait_ref = impl_trait_ref.subst(tcx, impl_substs);
 
-            let mut nested_goals = ecx.eq(goal.param_env, goal_trait_ref, impl_trait_ref)?;
+            ecx.eq(goal.param_env, goal_trait_ref, impl_trait_ref)?;
+
             let where_clause_bounds = tcx
                 .predicates_of(impl_def_id)
                 .instantiate(tcx, impl_substs)
                 .predicates
                 .into_iter()
                 .map(|pred| goal.with(tcx, pred));
-
-            nested_goals.extend(where_clause_bounds);
-            let match_impl_certainty = ecx.evaluate_all(nested_goals)?;
+            ecx.add_goals(where_clause_bounds);
 
             // In case the associated item is hidden due to specialization, we have to
             // return ambiguity this would otherwise be incomplete, resulting in
             // unsoundness during coherence (#105782).
             let Some(assoc_def) = fetch_eligible_assoc_item_def(
-                ecx.infcx,
+                ecx,
                 goal.param_env,
                 goal_trait_ref,
                 goal.predicate.def_id(),
                 impl_def_id
             )? else {
-                return ecx.make_canonical_response(match_impl_certainty.unify_and(Certainty::AMBIGUOUS));
+                return ecx.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
             };
 
             if !assoc_def.item.defaultness(tcx).has_value() {
@@ -240,8 +185,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 goal_trait_ref.def_id,
                 impl_substs,
             );
-            let substs = translate_substs(
-                ecx.infcx,
+            let substs = ecx.translate_substs(
                 goal.param_env,
                 impl_def_id,
                 impl_substs_with_gat,
@@ -262,7 +206,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 ty.map_bound(|ty| ty.into())
             };
 
-            ecx.eq_term_and_make_canonical_response(goal, match_impl_certainty, term.subst(tcx, substs))
+            ecx.eq(goal.param_env, goal.predicate.term, term.subst(tcx, substs))?;
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         })
     }
 
@@ -301,20 +246,31 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         bug!("`PointerLike` does not have an associated type: {:?}", goal);
     }
 
+    fn consider_builtin_fn_ptr_trait_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`FnPtr` does not have an associated type: {:?}", goal);
+    }
+
     fn consider_builtin_fn_trait_candidates(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
         goal_kind: ty::ClosureKind,
     ) -> QueryResult<'tcx> {
         let tcx = ecx.tcx();
-        let Some(tupled_inputs_and_output) =
-        structural_traits::extract_tupled_inputs_and_output_from_callable(
-            tcx,
-            goal.predicate.self_ty(),
-            goal_kind,
-        )? else {
-        return ecx.make_canonical_response(Certainty::AMBIGUOUS);
-    };
+        let tupled_inputs_and_output =
+            match structural_traits::extract_tupled_inputs_and_output_from_callable(
+                tcx,
+                goal.predicate.self_ty(),
+                goal_kind,
+            )? {
+                Some(tupled_inputs_and_output) => tupled_inputs_and_output,
+                None => {
+                    return ecx
+                        .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
+                }
+            };
         let output_is_sized_pred = tupled_inputs_and_output
             .map_bound(|(_, output)| tcx.at(DUMMY_SP).mk_trait_ref(LangItem::Sized, [output]));
 
@@ -378,27 +334,21 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                         LangItem::Sized,
                         [ty::GenericArg::from(goal.predicate.self_ty())],
                     ));
-
-                    let (_, is_sized_certainty) =
-                        ecx.evaluate_goal(goal.with(tcx, sized_predicate))?;
-                    return ecx.eq_term_and_make_canonical_response(
-                        goal,
-                        is_sized_certainty,
-                        tcx.types.unit,
-                    );
+                    ecx.add_goal(goal.with(tcx, sized_predicate));
+                    tcx.types.unit
                 }
 
                 ty::Adt(def, substs) if def.is_struct() => {
-                    match def.non_enum_variant().fields.last() {
+                    match def.non_enum_variant().fields.raw.last() {
                         None => tcx.types.unit,
                         Some(field_def) => {
                             let self_ty = field_def.ty(tcx, substs);
-                            let new_goal = goal.with(
+                            ecx.add_goal(goal.with(
                                 tcx,
                                 ty::Binder::dummy(goal.predicate.with_self_ty(tcx, self_ty)),
-                            );
-                            let (_, certainty) = ecx.evaluate_goal(new_goal)?;
-                            return ecx.make_canonical_response(certainty);
+                            ));
+                            return ecx
+                                .evaluate_added_goals_and_make_canonical_response(Certainty::Yes);
                         }
                     }
                 }
@@ -407,12 +357,12 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 ty::Tuple(elements) => match elements.last() {
                     None => tcx.types.unit,
                     Some(&self_ty) => {
-                        let new_goal = goal.with(
+                        ecx.add_goal(goal.with(
                             tcx,
                             ty::Binder::dummy(goal.predicate.with_self_ty(tcx, self_ty)),
-                        );
-                        let (_, certainty) = ecx.evaluate_goal(new_goal)?;
-                        return ecx.make_canonical_response(certainty);
+                        ));
+                        return ecx
+                            .evaluate_added_goals_and_make_canonical_response(Certainty::Yes);
                     }
                 },
 
@@ -425,7 +375,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 ),
             };
 
-            ecx.eq_term_and_make_canonical_response(goal, Certainty::Yes, metadata_ty)
+            ecx.eq(goal.param_env, goal.predicate.term, metadata_ty.into())?;
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         })
     }
 
@@ -512,7 +463,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
     fn consider_builtin_dyn_upcast_candidates(
         _ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
-    ) -> Vec<super::CanonicalResponse<'tcx>> {
+    ) -> Vec<CanonicalResponse<'tcx>> {
         bug!("`Unsize` does not have an associated type: {:?}", goal);
     }
 
@@ -520,8 +471,65 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
-        let discriminant = goal.predicate.self_ty().discriminant_ty(ecx.tcx());
-        ecx.probe(|ecx| ecx.eq_term_and_make_canonical_response(goal, Certainty::Yes, discriminant))
+        let self_ty = goal.predicate.self_ty();
+        let discriminant_ty = match *self_ty.kind() {
+            ty::Bool
+            | ty::Char
+            | ty::Int(..)
+            | ty::Uint(..)
+            | ty::Float(..)
+            | ty::Array(..)
+            | ty::RawPtr(..)
+            | ty::Ref(..)
+            | ty::FnDef(..)
+            | ty::FnPtr(..)
+            | ty::Closure(..)
+            | ty::Infer(ty::IntVar(..) | ty::FloatVar(..))
+            | ty::Generator(..)
+            | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
+            | ty::Never
+            | ty::Foreign(..)
+            | ty::Adt(_, _)
+            | ty::Str
+            | ty::Slice(_)
+            | ty::Dynamic(_, _, _)
+            | ty::Tuple(_)
+            | ty::Error(_) => self_ty.discriminant_ty(ecx.tcx()),
+
+            // We do not call `Ty::discriminant_ty` on alias, param, or placeholder
+            // types, which return `<self_ty as DiscriminantKind>::Discriminant`
+            // (or ICE in the case of placeholders). Projecting a type to itself
+            // is never really productive.
+            ty::Alias(_, _) | ty::Param(_) | ty::Placeholder(..) => {
+                return Err(NoSolution);
+            }
+
+            ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_))
+            | ty::Bound(..) => bug!(
+                "unexpected self ty `{:?}` when normalizing `<T as DiscriminantKind>::Discriminant`",
+                goal.predicate.self_ty()
+            ),
+        };
+
+        ecx.probe(|ecx| {
+            ecx.eq(goal.param_env, goal.predicate.term, discriminant_ty.into())?;
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        })
+    }
+
+    fn consider_builtin_destruct_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`Destruct` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_transmute_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`BikeshedIntrinsicFrom` does not have an associated type: {:?}", goal)
     }
 }
 
@@ -529,15 +537,15 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
 ///
 /// FIXME: We should merge these 3 implementations as it's likely that they otherwise
 /// diverge.
-#[instrument(level = "debug", skip(infcx, param_env), ret)]
+#[instrument(level = "debug", skip(ecx, param_env), ret)]
 fn fetch_eligible_assoc_item_def<'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    ecx: &EvalCtxt<'_, 'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     goal_trait_ref: ty::TraitRef<'tcx>,
     trait_assoc_def_id: DefId,
     impl_def_id: DefId,
 ) -> Result<Option<LeafDef>, NoSolution> {
-    let node_item = specialization_graph::assoc_def(infcx.tcx, impl_def_id, trait_assoc_def_id)
+    let node_item = specialization_graph::assoc_def(ecx.tcx(), impl_def_id, trait_assoc_def_id)
         .map_err(|ErrorGuaranteed { .. }| NoSolution)?;
 
     let eligible = if node_item.is_final() {
@@ -549,7 +557,7 @@ fn fetch_eligible_assoc_item_def<'tcx>(
         // transmute checking and polymorphic MIR optimizations could
         // get a result which isn't correct for all monomorphizations.
         if param_env.reveal() == Reveal::All {
-            let poly_trait_ref = infcx.resolve_vars_if_possible(goal_trait_ref);
+            let poly_trait_ref = ecx.resolve_vars_if_possible(goal_trait_ref);
             !poly_trait_ref.still_further_specializable()
         } else {
             debug!(?node_item.item.def_id, "not eligible due to default");
diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs b/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs
index 86b13c05f..d1b4fa554 100644
--- a/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs
+++ b/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs
@@ -8,12 +8,10 @@
 //!
 //! FIXME(@lcnr): Write that section, feel free to ping me if you need help here
 //! before then or if I still haven't done that before January 2023.
-use super::overflow::OverflowData;
 use super::StackDepth;
-use crate::solve::{CanonicalGoal, QueryResult};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_index::vec::IndexVec;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::traits::solve::{CanonicalGoal, QueryResult};
 
 rustc_index::newtype_index! {
     pub struct EntryIndex {}
@@ -98,26 +96,3 @@ impl<'tcx> ProvisionalCache<'tcx> {
         self.entries[entry_index].response
     }
 }
-
-pub(super) fn try_move_finished_goal_to_global_cache<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    overflow_data: &mut OverflowData,
-    stack: &IndexVec<super::StackDepth, super::StackElem<'tcx>>,
-    goal: CanonicalGoal<'tcx>,
-    response: QueryResult<'tcx>,
-) {
-    // We move goals to the global cache if we either did not hit an overflow or if it's
-    // the root goal as that will now always hit the same overflow limit.
-    //
-    // NOTE: We cannot move any non-root goals to the global cache even if their final result
-    // isn't impacted by the overflow as that goal still has unstable query dependencies
-    // because it didn't go its full depth.
-    //
-    // FIXME(@lcnr): We could still cache subtrees which are not impacted by overflow though.
-    // Tracking that info correctly isn't trivial, so I haven't implemented it for now.
-    let should_cache_globally = !overflow_data.did_overflow() || stack.is_empty();
-    if should_cache_globally {
-        // FIXME: move the provisional entry to the global cache.
-        let _ = (tcx, goal, response);
-    }
-}
diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs
index c7eb8de65..050269fa9 100644
--- a/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs
@@ -1,15 +1,19 @@
 mod cache;
 mod overflow;
 
+pub(super) use overflow::OverflowHandler;
+
 use self::cache::ProvisionalEntry;
-use super::{CanonicalGoal, Certainty, MaybeCause, QueryResult};
-pub(super) use crate::solve::search_graph::overflow::OverflowHandler;
 use cache::ProvisionalCache;
 use overflow::OverflowData;
 use rustc_index::vec::IndexVec;
+use rustc_middle::dep_graph::DepKind;
+use rustc_middle::traits::solve::{CanonicalGoal, Certainty, MaybeCause, QueryResult};
 use rustc_middle::ty::TyCtxt;
 use std::{collections::hash_map::Entry, mem};
 
+use super::SolverMode;
+
 rustc_index::newtype_index! {
     pub struct StackDepth {}
 }
@@ -20,6 +24,7 @@ struct StackElem<'tcx> {
 }
 
 pub(super) struct SearchGraph<'tcx> {
+    mode: SolverMode,
     /// The stack of goals currently being computed.
     ///
     /// An element is *deeper* in the stack if its index is *lower*.
@@ -29,24 +34,43 @@ pub(super) struct SearchGraph<'tcx> {
 }
 
 impl<'tcx> SearchGraph<'tcx> {
-    pub(super) fn new(tcx: TyCtxt<'tcx>) -> SearchGraph<'tcx> {
+    pub(super) fn new(tcx: TyCtxt<'tcx>, mode: SolverMode) -> SearchGraph<'tcx> {
         Self {
+            mode,
             stack: Default::default(),
             overflow_data: OverflowData::new(tcx),
             provisional_cache: ProvisionalCache::empty(),
         }
     }
 
+    pub(super) fn solver_mode(&self) -> SolverMode {
+        self.mode
+    }
+
+    /// We do not use the global cache during coherence.
+    ///
+    /// The trait solver behavior is different for coherence
+    /// so we would have to add the solver mode to the cache key.
+    /// This is probably not worth it as trait solving during
+    /// coherence tends to already be incredibly fast.
+    ///
+    /// We could add another global cache for coherence instead,
+    /// but that's effort so let's only do it if necessary.
+    pub(super) fn should_use_global_cache(&self) -> bool {
+        match self.mode {
+            SolverMode::Normal => true,
+            SolverMode::Coherence => false,
+        }
+    }
+
     pub(super) fn is_empty(&self) -> bool {
-        self.stack.is_empty()
-            && self.provisional_cache.is_empty()
-            && !self.overflow_data.did_overflow()
+        self.stack.is_empty() && self.provisional_cache.is_empty()
     }
 
     /// Whether we're currently in a cycle. This should only be used
     /// for debug assertions.
     pub(super) fn in_cycle(&self) -> bool {
-        if let Some(stack_depth) = self.stack.last() {
+        if let Some(stack_depth) = self.stack.last_index() {
             // Either the current goal on the stack is the root of a cycle...
             if self.stack[stack_depth].has_been_used {
                 return true;
@@ -70,8 +94,6 @@ impl<'tcx> SearchGraph<'tcx> {
         tcx: TyCtxt<'tcx>,
         goal: CanonicalGoal<'tcx>,
     ) -> Result<(), QueryResult<'tcx>> {
-        // FIXME: start by checking the global cache
-
         // Look at the provisional cache to check for cycles.
         let cache = &mut self.provisional_cache;
         match cache.lookup_table.entry(goal) {
@@ -131,7 +153,7 @@ impl<'tcx> SearchGraph<'tcx> {
     /// coinductive cycles.
     ///
     /// When we encounter a coinductive cycle, we have to prove the final result of that cycle
-    /// while we are still computing that result. Because of this we continously recompute the
+    /// while we are still computing that result. Because of this we continuously recompute the
     /// cycle until the result of the previous iteration is equal to the final result, at which
     /// point we are done.
     ///
@@ -139,10 +161,9 @@ impl<'tcx> SearchGraph<'tcx> {
     /// updated the provisional cache and we have to recompute the current goal.
     ///
     /// FIXME: Refer to the rustc-dev-guide entry once it exists.
-    #[instrument(level = "debug", skip(self, tcx, actual_goal), ret)]
+    #[instrument(level = "debug", skip(self, actual_goal), ret)]
     fn try_finalize_goal(
         &mut self,
-        tcx: TyCtxt<'tcx>,
         actual_goal: CanonicalGoal<'tcx>,
         response: QueryResult<'tcx>,
     ) -> bool {
@@ -176,72 +197,90 @@ impl<'tcx> SearchGraph<'tcx> {
             self.stack.push(StackElem { goal, has_been_used: false });
             false
         } else {
-            self.try_move_finished_goal_to_global_cache(tcx, stack_elem);
             true
         }
     }
 
-    fn try_move_finished_goal_to_global_cache(
+    pub(super) fn with_new_goal(
         &mut self,
         tcx: TyCtxt<'tcx>,
-        stack_elem: StackElem<'tcx>,
-    ) {
-        let StackElem { goal, .. } = stack_elem;
+        canonical_goal: CanonicalGoal<'tcx>,
+        mut loop_body: impl FnMut(&mut Self) -> QueryResult<'tcx>,
+    ) -> QueryResult<'tcx> {
+        if self.should_use_global_cache() {
+            if let Some(result) = tcx.new_solver_evaluation_cache.get(&canonical_goal, tcx) {
+                debug!(?canonical_goal, ?result, "cache hit");
+                return result;
+            }
+        }
+
+        match self.try_push_stack(tcx, canonical_goal) {
+            Ok(()) => {}
+            // Our goal is already on the stack, eager return.
+            Err(response) => return response,
+        }
+
+        // This is for global caching, so we properly track query dependencies.
+        // Everything that affects the `Result` should be performed within this
+        // `with_anon_task` closure.
+        let (result, dep_node) = tcx.dep_graph.with_anon_task(tcx, DepKind::TraitSelect, || {
+            self.repeat_while_none(
+                |this| {
+                    let result = this.deal_with_overflow(tcx, canonical_goal);
+                    let _ = this.stack.pop().unwrap();
+                    result
+                },
+                |this| {
+                    let result = loop_body(this);
+                    this.try_finalize_goal(canonical_goal, result).then(|| result)
+                },
+            )
+        });
+
         let cache = &mut self.provisional_cache;
-        let provisional_entry_index = *cache.lookup_table.get(&goal).unwrap();
+        let provisional_entry_index = *cache.lookup_table.get(&canonical_goal).unwrap();
         let provisional_entry = &mut cache.entries[provisional_entry_index];
         let depth = provisional_entry.depth;
 
         // If not, we're done with this goal.
         //
         // Check whether that this goal doesn't depend on a goal deeper on the stack
-        // and if so, move it and all nested goals to the global cache.
+        // and if so, move it to the global cache.
         //
         // Note that if any nested goal were to depend on something deeper on the stack,
         // this would have also updated the depth of the current goal.
         if depth == self.stack.next_index() {
-            for (i, entry) in cache.entries.drain_enumerated(provisional_entry_index.index()..) {
+            // If the current goal is the head of a cycle, we drop all other
+            // cycle participants without moving them to the global cache.
+            let other_cycle_participants = provisional_entry_index.index() + 1;
+            for (i, entry) in cache.entries.drain_enumerated(other_cycle_participants..) {
                 let actual_index = cache.lookup_table.remove(&entry.goal);
                 debug_assert_eq!(Some(i), actual_index);
                 debug_assert!(entry.depth == depth);
-                cache::try_move_finished_goal_to_global_cache(
-                    tcx,
-                    &mut self.overflow_data,
-                    &self.stack,
-                    entry.goal,
-                    entry.response,
-                );
             }
-        }
-    }
 
-    pub(super) fn with_new_goal(
-        &mut self,
-        tcx: TyCtxt<'tcx>,
-        canonical_goal: CanonicalGoal<'tcx>,
-        mut loop_body: impl FnMut(&mut Self) -> QueryResult<'tcx>,
-    ) -> QueryResult<'tcx> {
-        match self.try_push_stack(tcx, canonical_goal) {
-            Ok(()) => {}
-            // Our goal is already on the stack, eager return.
-            Err(response) => return response,
+            let current_goal = cache.entries.pop().unwrap();
+            let actual_index = cache.lookup_table.remove(&current_goal.goal);
+            debug_assert_eq!(Some(provisional_entry_index), actual_index);
+            debug_assert!(current_goal.depth == depth);
+
+            // We move the root goal to the global cache if we either did not hit an overflow or if it's
+            // the root goal as that will now always hit the same overflow limit.
+            //
+            // NOTE: We cannot move any non-root goals to the global cache. When replaying the root goal's
+            // dependencies, our non-root goal may no longer appear as child of the root goal.
+            //
+            // See https://github.com/rust-lang/rust/pull/108071 for some additional context.
+            let can_cache = !self.overflow_data.did_overflow() || self.stack.is_empty();
+            if self.should_use_global_cache() && can_cache {
+                tcx.new_solver_evaluation_cache.insert(
+                    current_goal.goal,
+                    dep_node,
+                    current_goal.response,
+                );
+            }
         }
 
-        self.repeat_while_none(
-            |this| {
-                let result = this.deal_with_overflow(tcx, canonical_goal);
-                let stack_elem = this.stack.pop().unwrap();
-                this.try_move_finished_goal_to_global_cache(tcx, stack_elem);
-                result
-            },
-            |this| {
-                let result = loop_body(this);
-                if this.try_finalize_goal(tcx, canonical_goal, result) {
-                    Some(result)
-                } else {
-                    None
-                }
-            },
-        )
+        result
     }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs b/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs
index 56409b060..e0a2e0c5c 100644
--- a/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs
+++ b/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs
@@ -1,10 +1,11 @@
 use rustc_infer::infer::canonical::Canonical;
 use rustc_infer::traits::query::NoSolution;
+use rustc_middle::traits::solve::{Certainty, MaybeCause, QueryResult};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Limit;
 
 use super::SearchGraph;
-use crate::solve::{response_no_constraints, Certainty, EvalCtxt, MaybeCause, QueryResult};
+use crate::solve::{response_no_constraints, EvalCtxt};
 
 /// When detecting a solver overflow, we return ambiguity. Overflow can be
 /// *hidden* by either a fatal error in an **AND** or a trivial success in an **OR**.
@@ -44,7 +45,7 @@ impl OverflowData {
     /// Updating the current limit when hitting overflow.
     fn deal_with_overflow(&mut self) {
         // When first hitting overflow we reduce the overflow limit
-        // for all future goals to prevent hangs if there's an exponental
+        // for all future goals to prevent hangs if there's an exponential
         // blowup.
         self.current_limit.0 = self.default_limit.0 / 8;
     }
@@ -72,6 +73,27 @@ pub(in crate::solve) trait OverflowHandler<'tcx> {
         self.search_graph().overflow_data.deal_with_overflow();
         on_overflow(self)
     }
+
+    // Increment the `additional_depth` by one and evaluate `body`, or `on_overflow`
+    // if the depth is overflown.
+    fn with_incremented_depth<T>(
+        &mut self,
+        on_overflow: impl FnOnce(&mut Self) -> T,
+        body: impl FnOnce(&mut Self) -> T,
+    ) -> T {
+        let depth = self.search_graph().stack.len();
+        self.search_graph().overflow_data.additional_depth += 1;
+
+        let result = if self.search_graph().overflow_data.has_overflow(depth) {
+            self.search_graph().overflow_data.deal_with_overflow();
+            on_overflow(self)
+        } else {
+            body(self)
+        };
+
+        self.search_graph().overflow_data.additional_depth -= 1;
+        result
+    }
 }
 
 impl<'tcx> OverflowHandler<'tcx> for EvalCtxt<'_, 'tcx> {
diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
index 5c499c36e..abd11a15a 100644
--- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs
+++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
@@ -1,25 +1,26 @@
 //! Dealing with trait goals, i.e. `T: Trait<'a, U>`.
 
-use std::iter;
-
-use super::assembly;
-use super::{CanonicalResponse, Certainty, EvalCtxt, Goal, QueryResult};
+use super::assembly::{self, structural_traits};
+use super::{EvalCtxt, SolverMode};
 use rustc_hir::def_id::DefId;
-use rustc_hir::LangItem;
+use rustc_hir::{LangItem, Movability};
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::util::supertraits;
-use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_middle::traits::solve::{CanonicalResponse, Certainty, Goal, QueryResult};
+use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams, TreatProjections};
 use rustc_middle::ty::{self, ToPredicate, Ty, TyCtxt};
 use rustc_middle::ty::{TraitPredicate, TypeVisitableExt};
 use rustc_span::DUMMY_SP;
 
-pub mod structural_traits;
-
 impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
     fn self_ty(self) -> Ty<'tcx> {
         self.self_ty()
     }
 
+    fn trait_ref(self, _: TyCtxt<'tcx>) -> ty::TraitRef<'tcx> {
+        self.trait_ref
+    }
+
     fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
         self.with_self_ty(tcx, self_ty)
     }
@@ -36,27 +37,44 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         let tcx = ecx.tcx();
 
         let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap();
-        let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsPlaceholder };
-        if iter::zip(goal.predicate.trait_ref.substs, impl_trait_ref.skip_binder().substs)
-            .any(|(goal, imp)| !drcx.generic_args_may_unify(goal, imp))
-        {
+        let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::ForLookup };
+        if !drcx.substs_refs_may_unify(
+            goal.predicate.trait_ref.substs,
+            impl_trait_ref.skip_binder().substs,
+        ) {
             return Err(NoSolution);
         }
 
+        let impl_polarity = tcx.impl_polarity(impl_def_id);
+        // An upper bound of the certainty of this goal, used to lower the certainty
+        // of reservation impl to ambiguous during coherence.
+        let maximal_certainty = match impl_polarity {
+            ty::ImplPolarity::Positive | ty::ImplPolarity::Negative => {
+                match impl_polarity == goal.predicate.polarity {
+                    true => Certainty::Yes,
+                    false => return Err(NoSolution),
+                }
+            }
+            ty::ImplPolarity::Reservation => match ecx.solver_mode() {
+                SolverMode::Normal => return Err(NoSolution),
+                SolverMode::Coherence => Certainty::AMBIGUOUS,
+            },
+        };
+
         ecx.probe(|ecx| {
             let impl_substs = ecx.fresh_substs_for_item(impl_def_id);
             let impl_trait_ref = impl_trait_ref.subst(tcx, impl_substs);
 
-            let mut nested_goals =
-                ecx.eq(goal.param_env, goal.predicate.trait_ref, impl_trait_ref)?;
+            ecx.eq(goal.param_env, goal.predicate.trait_ref, impl_trait_ref)?;
             let where_clause_bounds = tcx
                 .predicates_of(impl_def_id)
                 .instantiate(tcx, impl_substs)
                 .predicates
                 .into_iter()
                 .map(|pred| goal.with(tcx, pred));
-            nested_goals.extend(where_clause_bounds);
-            ecx.evaluate_all_and_make_canonical_response(nested_goals)
+            ecx.add_goals(where_clause_bounds);
+
+            ecx.evaluate_added_goals_and_make_canonical_response(maximal_certainty)
         })
     }
 
@@ -73,13 +91,13 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             ecx.probe(|ecx| {
                 let assumption_trait_pred =
                     ecx.instantiate_binder_with_infer(poly_trait_pred);
-                let mut nested_goals = ecx.eq(
+                ecx.eq(
                     goal.param_env,
                     goal.predicate.trait_ref,
                     assumption_trait_pred.trait_ref,
                 )?;
-                nested_goals.extend(requirements);
-                ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                ecx.add_goals(requirements);
+                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             })
         } else {
             Err(NoSolution)
@@ -98,7 +116,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             ecx.probe(|ecx| {
                 let assumption_trait_pred =
                     ecx.instantiate_binder_with_infer(poly_trait_pred);
-                let mut nested_goals = ecx.eq(
+                ecx.eq(
                     goal.param_env,
                     goal.predicate.trait_ref,
                     assumption_trait_pred.trait_ref,
@@ -108,9 +126,9 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                 let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else {
                     bug!("expected object type in `consider_object_bound_candidate`");
                 };
-                nested_goals.extend(
+                ecx.add_goals(
                     structural_traits::predicates_for_object_candidate(
-                        ecx,
+                        &ecx,
                         goal.param_env,
                         goal.predicate.trait_ref,
                         bounds,
@@ -118,8 +136,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                     .into_iter()
                     .map(|pred| goal.with(tcx, pred)),
                 );
-
-                ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             })
         } else {
             Err(NoSolution)
@@ -130,18 +147,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
-        // This differs from the current stable behavior and
-        // fixes #84857. Due to breakage found via crater, we
-        // currently instead lint patterns which can be used to
-        // exploit this unsoundness on stable, see #93367 for
-        // more details.
-        if let Some(def_id) = ecx.tcx().find_map_relevant_impl(
-            goal.predicate.def_id(),
-            goal.predicate.self_ty(),
-            Some,
-        ) {
-            debug!(?def_id, ?goal, "disqualified auto-trait implementation");
-            return Err(NoSolution);
+        if let Some(result) = ecx.disqualify_auto_trait_candidate_due_to_possible_impl(goal) {
+            return result;
         }
 
         ecx.probe_and_evaluate_goal_for_constituent_tys(
@@ -160,9 +167,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             let nested_obligations = tcx
                 .predicates_of(goal.predicate.def_id())
                 .instantiate(tcx, goal.predicate.trait_ref.substs);
-            ecx.evaluate_all_and_make_canonical_response(
-                nested_obligations.predicates.into_iter().map(|p| goal.with(tcx, p)).collect(),
-            )
+            ecx.add_goals(nested_obligations.predicates.into_iter().map(|p| goal.with(tcx, p)));
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         })
     }
 
@@ -191,19 +197,28 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
         if goal.predicate.self_ty().has_non_region_infer() {
-            return ecx.make_canonical_response(Certainty::AMBIGUOUS);
+            return ecx.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
         }
 
         let tcx = ecx.tcx();
         let self_ty = tcx.erase_regions(goal.predicate.self_ty());
 
         if let Ok(layout) = tcx.layout_of(goal.param_env.and(self_ty))
-            &&  let usize_layout = tcx.layout_of(ty::ParamEnv::empty().and(tcx.types.usize)).unwrap().layout
-            && layout.layout.size() == usize_layout.size()
-            && layout.layout.align().abi == usize_layout.align().abi
+            && layout.layout.is_pointer_like(&tcx.data_layout)
         {
             // FIXME: We could make this faster by making a no-constraints response
-            ecx.make_canonical_response(Certainty::Yes)
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        } else {
+            Err(NoSolution)
+        }
+    }
+
+    fn consider_builtin_fn_ptr_trait_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        if let ty::FnPtr(..) = goal.predicate.self_ty().kind() {
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         } else {
             Err(NoSolution)
         }
@@ -215,14 +230,18 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         goal_kind: ty::ClosureKind,
     ) -> QueryResult<'tcx> {
         let tcx = ecx.tcx();
-        let Some(tupled_inputs_and_output) =
-            structural_traits::extract_tupled_inputs_and_output_from_callable(
+        let tupled_inputs_and_output =
+            match structural_traits::extract_tupled_inputs_and_output_from_callable(
                 tcx,
                 goal.predicate.self_ty(),
                 goal_kind,
-            )? else {
-            return ecx.make_canonical_response(Certainty::AMBIGUOUS);
-        };
+            )? {
+                Some(a) => a,
+                None => {
+                    return ecx
+                        .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
+                }
+            };
         let output_is_sized_pred = tupled_inputs_and_output
             .map_bound(|(_, output)| tcx.at(DUMMY_SP).mk_trait_ref(LangItem::Sized, [output]));
 
@@ -241,7 +260,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
         if let ty::Tuple(..) = goal.predicate.self_ty().kind() {
-            ecx.make_canonical_response(Certainty::Yes)
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         } else {
             Err(NoSolution)
         }
@@ -251,7 +270,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         ecx: &mut EvalCtxt<'_, 'tcx>,
         _goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
-        ecx.make_canonical_response(Certainty::Yes)
+        ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
     fn consider_builtin_future_candidate(
@@ -271,7 +290,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         // Async generator unconditionally implement `Future`
         // Technically, we need to check that the future output type is Sized,
         // but that's already proven by the generator being WF.
-        ecx.make_canonical_response(Certainty::Yes)
+        ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
     fn consider_builtin_generator_candidate(
@@ -311,7 +330,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         let a_ty = goal.predicate.self_ty();
         let b_ty = goal.predicate.trait_ref.substs.type_at(1);
         if b_ty.is_ty_var() {
-            return ecx.make_canonical_response(Certainty::AMBIGUOUS);
+            return ecx.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
         }
         ecx.probe(|ecx| {
             match (a_ty.kind(), b_ty.kind()) {
@@ -320,7 +339,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                     // Dyn upcasting is handled separately, since due to upcasting,
                     // when there are two supertraits that differ by substs, we
                     // may return more than one query response.
-                    return Err(NoSolution);
+                    Err(NoSolution)
                 }
                 // `T` -> `dyn Trait` unsizing
                 (_, &ty::Dynamic(data, region, ty::Dyn)) => {
@@ -335,29 +354,26 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                     let Some(sized_def_id) = tcx.lang_items().sized_trait() else {
                         return Err(NoSolution);
                     };
-                    let nested_goals: Vec<_> = data
-                        .iter()
-                        // Check that the type implements all of the predicates of the def-id.
-                        // (i.e. the principal, all of the associated types match, and any auto traits)
-                        .map(|pred| goal.with(tcx, pred.with_self_ty(tcx, a_ty)))
-                        .chain([
-                            // The type must be Sized to be unsized.
-                            goal.with(
-                                tcx,
-                                ty::Binder::dummy(tcx.mk_trait_ref(sized_def_id, [a_ty])),
-                            ),
-                            // The type must outlive the lifetime of the `dyn` we're unsizing into.
-                            goal.with(tcx, ty::Binder::dummy(ty::OutlivesPredicate(a_ty, region))),
-                        ])
-                        .collect();
-
-                    ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                    // Check that the type implements all of the predicates of the def-id.
+                    // (i.e. the principal, all of the associated types match, and any auto traits)
+                    ecx.add_goals(
+                        data.iter().map(|pred| goal.with(tcx, pred.with_self_ty(tcx, a_ty))),
+                    );
+                    // The type must be Sized to be unsized.
+                    ecx.add_goal(
+                        goal.with(tcx, ty::Binder::dummy(tcx.mk_trait_ref(sized_def_id, [a_ty]))),
+                    );
+                    // The type must outlive the lifetime of the `dyn` we're unsizing into.
+                    ecx.add_goal(
+                        goal.with(tcx, ty::Binder::dummy(ty::OutlivesPredicate(a_ty, region))),
+                    );
+                    ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
                 }
                 // `[T; n]` -> `[T]` unsizing
                 (&ty::Array(a_elem_ty, ..), &ty::Slice(b_elem_ty)) => {
                     // We just require that the element type stays the same
-                    let nested_goals = ecx.eq(goal.param_env, a_elem_ty, b_elem_ty)?;
-                    ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                    ecx.eq(goal.param_env, a_elem_ty, b_elem_ty)?;
+                    ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
                 }
                 // Struct unsizing `Struct<T>` -> `Struct<U>` where `T: Unsize<U>`
                 (&ty::Adt(a_def, a_substs), &ty::Adt(b_def, b_substs))
@@ -373,6 +389,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                     let tail_field = a_def
                         .non_enum_variant()
                         .fields
+                        .raw
                         .last()
                         .expect("expected unsized ADT to have a tail field");
                     let tail_field_ty = tcx.type_of(tail_field.did);
@@ -391,15 +408,14 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
 
                     // Finally, we require that `TailA: Unsize<TailB>` for the tail field
                     // types.
-                    let mut nested_goals = ecx.eq(goal.param_env, unsized_a_ty, b_ty)?;
-                    nested_goals.push(goal.with(
+                    ecx.eq(goal.param_env, unsized_a_ty, b_ty)?;
+                    ecx.add_goal(goal.with(
                         tcx,
                         ty::Binder::dummy(
                             tcx.mk_trait_ref(goal.predicate.def_id(), [a_tail_ty, b_tail_ty]),
                         ),
                     ));
-
-                    ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                    ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
                 }
                 // Tuple unsizing `(.., T)` -> `(.., U)` where `T: Unsize<U>`
                 (&ty::Tuple(a_tys), &ty::Tuple(b_tys))
@@ -411,17 +427,16 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                     // Substitute just the tail field of B., and require that they're equal.
                     let unsized_a_ty =
                         tcx.mk_tup_from_iter(a_rest_tys.iter().chain([b_last_ty]).copied());
-                    let mut nested_goals = ecx.eq(goal.param_env, unsized_a_ty, b_ty)?;
+                    ecx.eq(goal.param_env, unsized_a_ty, b_ty)?;
 
                     // Similar to ADTs, require that the rest of the fields are equal.
-                    nested_goals.push(goal.with(
+                    ecx.add_goal(goal.with(
                         tcx,
                         ty::Binder::dummy(
                             tcx.mk_trait_ref(goal.predicate.def_id(), [*a_last_ty, *b_last_ty]),
                         ),
                     ));
-
-                    ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                    ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
                 }
                 _ => Err(NoSolution),
             }
@@ -471,12 +486,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                 let new_a_ty = tcx.mk_dynamic(new_a_data, b_region, ty::Dyn);
 
                 // We also require that A's lifetime outlives B's lifetime.
-                let mut nested_obligations = ecx.eq(goal.param_env, new_a_ty, b_ty)?;
-                nested_obligations.push(
+                ecx.eq(goal.param_env, new_a_ty, b_ty)?;
+                ecx.add_goal(
                     goal.with(tcx, ty::Binder::dummy(ty::OutlivesPredicate(a_region, b_region))),
                 );
-
-                ecx.evaluate_all_and_make_canonical_response(nested_obligations)
+                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             })
         };
 
@@ -510,11 +524,145 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         _goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
         // `DiscriminantKind` is automatically implemented for every type.
-        ecx.make_canonical_response(Certainty::Yes)
+        ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+    }
+
+    fn consider_builtin_destruct_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        if !goal.param_env.is_const() {
+            // `Destruct` is automatically implemented for every type in
+            // non-const environments.
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        } else {
+            // FIXME(-Ztrait-solver=next): Implement this when we get const working in the new solver
+            Err(NoSolution)
+        }
+    }
+
+    fn consider_builtin_transmute_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        // `rustc_transmute` does not have support for type or const params
+        if goal.has_non_region_placeholders() {
+            return Err(NoSolution);
+        }
+
+        // Erase regions because we compute layouts in `rustc_transmute`,
+        // which will ICE for region vars.
+        let substs = ecx.tcx().erase_regions(goal.predicate.trait_ref.substs);
+
+        let Some(assume) = rustc_transmute::Assume::from_const(
+            ecx.tcx(),
+            goal.param_env,
+            substs.const_at(3),
+        ) else {
+            return Err(NoSolution);
+        };
+
+        let certainty = ecx.is_transmutable(
+            rustc_transmute::Types { dst: substs.type_at(0), src: substs.type_at(1) },
+            substs.type_at(2),
+            assume,
+        )?;
+        ecx.evaluate_added_goals_and_make_canonical_response(certainty)
     }
 }
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
+    // Return `Some` if there is an impl (built-in or user provided) that may
+    // hold for the self type of the goal, which for coherence and soundness
+    // purposes must disqualify the built-in auto impl assembled by considering
+    // the type's constituent types.
+    fn disqualify_auto_trait_candidate_due_to_possible_impl(
+        &mut self,
+        goal: Goal<'tcx, TraitPredicate<'tcx>>,
+    ) -> Option<QueryResult<'tcx>> {
+        let self_ty = goal.predicate.self_ty();
+        match *self_ty.kind() {
+            // Stall int and float vars until they are resolved to a concrete
+            // numerical type. That's because the check for impls below treats
+            // int vars as matching any impl. Even if we filtered such impls,
+            // we probably don't want to treat an `impl !AutoTrait for i32` as
+            // disqualifying the built-in auto impl for `i64: AutoTrait` either.
+            ty::Infer(ty::IntVar(_) | ty::FloatVar(_)) => {
+                Some(self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS))
+            }
+
+            // These types cannot be structurally decomposed into constitutent
+            // types, and therefore have no built-in auto impl.
+            ty::Dynamic(..)
+            | ty::Param(..)
+            | ty::Foreign(..)
+            | ty::Alias(ty::Projection, ..)
+            | ty::Placeholder(..) => Some(Err(NoSolution)),
+
+            ty::Infer(_) | ty::Bound(_, _) => bug!("unexpected type `{self_ty}`"),
+
+            // Generators have one special built-in candidate, `Unpin`, which
+            // takes precedence over the structural auto trait candidate being
+            // assembled.
+            ty::Generator(_, _, movability)
+                if Some(goal.predicate.def_id()) == self.tcx().lang_items().unpin_trait() =>
+            {
+                match movability {
+                    Movability::Static => Some(Err(NoSolution)),
+                    Movability::Movable => {
+                        Some(self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes))
+                    }
+                }
+            }
+
+            // For rigid types, any possible implementation that could apply to
+            // the type (even if after unification and processing nested goals
+            // it does not hold) will disqualify the built-in auto impl.
+            //
+            // This differs from the current stable behavior and fixes #84857.
+            // Due to breakage found via crater, we currently instead lint
+            // patterns which can be used to exploit this unsoundness on stable,
+            // see #93367 for more details.
+            ty::Bool
+            | ty::Char
+            | ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Str
+            | ty::Array(_, _)
+            | ty::Slice(_)
+            | ty::RawPtr(_)
+            | ty::Ref(_, _, _)
+            | ty::FnDef(_, _)
+            | ty::FnPtr(_)
+            | ty::Closure(_, _)
+            | ty::Generator(_, _, _)
+            | ty::GeneratorWitness(_)
+            | ty::GeneratorWitnessMIR(_, _)
+            | ty::Never
+            | ty::Tuple(_)
+            | ty::Adt(_, _)
+            // FIXME: Handling opaques here is kinda sus. Especially because we
+            // simplify them to PlaceholderSimplifiedType.
+            | ty::Alias(ty::Opaque, _) => {
+                if let Some(def_id) = self.tcx().find_map_relevant_impl(
+                    goal.predicate.def_id(),
+                    goal.predicate.self_ty(),
+                    TreatProjections::NextSolverLookup,
+                    Some,
+                ) {
+                    debug!(?def_id, ?goal, "disqualified auto-trait implementation");
+                    // No need to actually consider the candidate here,
+                    // since we do that in `consider_impl_candidate`.
+                    return Some(Err(NoSolution));
+                } else {
+                    None
+                }
+            }
+            ty::Error(_) => None,
+        }
+    }
+
     /// Convenience function for traits that are structural, i.e. that only
     /// have nested subgoals that only change the self type. Unlike other
     /// evaluate-like helpers, this does a probe, so it doesn't need to be
@@ -524,26 +672,28 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         goal: Goal<'tcx, TraitPredicate<'tcx>>,
         constituent_tys: impl Fn(&EvalCtxt<'_, 'tcx>, Ty<'tcx>) -> Result<Vec<Ty<'tcx>>, NoSolution>,
     ) -> QueryResult<'tcx> {
-        self.probe(|this| {
-            this.evaluate_all_and_make_canonical_response(
-                constituent_tys(this, goal.predicate.self_ty())?
+        self.probe(|ecx| {
+            ecx.add_goals(
+                constituent_tys(ecx, goal.predicate.self_ty())?
                     .into_iter()
                     .map(|ty| {
                         goal.with(
-                            this.tcx(),
-                            ty::Binder::dummy(goal.predicate.with_self_ty(this.tcx(), ty)),
+                            ecx.tcx(),
+                            ty::Binder::dummy(goal.predicate.with_self_ty(ecx.tcx(), ty)),
                         )
                     })
-                    .collect(),
-            )
+                    .collect::<Vec<_>>(),
+            );
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         })
     }
 
+    #[instrument(level = "debug", skip(self))]
     pub(super) fn compute_trait_goal(
         &mut self,
         goal: Goal<'tcx, TraitPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
         let candidates = self.assemble_and_evaluate_candidates(goal);
-        self.merge_candidates_and_discard_reservation_impls(candidates)
+        self.merge_candidates(candidates)
     }
 }
diff --git a/compiler/rustc_trait_selection/src/traits/auto_trait.rs b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
index 1fb8659bb..182d995c4 100644
--- a/compiler/rustc_trait_selection/src/traits/auto_trait.rs
+++ b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
@@ -7,8 +7,8 @@ use crate::errors::UnableToConstructConstantValue;
 use crate::infer::region_constraints::{Constraint, RegionConstraintData};
 use crate::infer::InferCtxt;
 use crate::traits::project::ProjectAndUnifyResult;
+use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_middle::mir::interpret::ErrorHandled;
-use rustc_middle::ty::fold::{TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{ImplPolarity, Region, RegionVid};
 
@@ -179,13 +179,15 @@ impl<'tcx> AutoTraitFinder<'tcx> {
         // At this point, we already have all of the bounds we need. FulfillmentContext is used
         // to store all of the necessary region/lifetime bounds in the InferContext, as well as
         // an additional sanity check.
-        let errors =
-            super::fully_solve_bound(&infcx, ObligationCause::dummy(), full_env, ty, trait_did);
+        let ocx = ObligationCtxt::new(&infcx);
+        ocx.register_bound(ObligationCause::dummy(), full_env, ty, trait_did);
+        let errors = ocx.select_all_or_error();
         if !errors.is_empty() {
             panic!("Unable to fulfill trait {:?} for '{:?}': {:?}", trait_did, ty, errors);
         }
 
-        infcx.process_registered_region_obligations(&Default::default(), full_env);
+        let outlives_env = OutlivesEnvironment::new(full_env);
+        infcx.process_registered_region_obligations(&outlives_env);
 
         let region_data =
             infcx.inner.borrow_mut().unwrap_region_constraints().region_constraint_data().clone();
@@ -232,7 +234,7 @@ impl<'tcx> AutoTraitFinder<'tcx> {
     /// constructed once for a given type. As part of the construction process, the `ParamEnv` will
     /// have any supertrait bounds normalized -- e.g., if we have a type `struct Foo<T: Copy>`, the
     /// `ParamEnv` will contain `T: Copy` and `T: Clone`, since `Copy: Clone`. When we construct our
-    /// own `ParamEnv`, we need to do this ourselves, through `traits::elaborate_predicates`, or
+    /// own `ParamEnv`, we need to do this ourselves, through `traits::elaborate`, or
     /// else `SelectionContext` will choke on the missing predicates. However, this should never
     /// show up in the final synthesized generics: we don't want our generated docs page to contain
     /// something like `T: Copy + Clone`, as that's redundant. Therefore, we keep track of a
@@ -344,11 +346,8 @@ impl<'tcx> AutoTraitFinder<'tcx> {
                 _ => panic!("Unexpected error for '{:?}': {:?}", ty, result),
             };
 
-            let normalized_preds = elaborate_predicates(
-                tcx,
-                computed_preds.clone().chain(user_computed_preds.iter().cloned()),
-            )
-            .map(|o| o.predicate);
+            let normalized_preds =
+                elaborate(tcx, computed_preds.clone().chain(user_computed_preds.iter().cloned()));
             new_env = ty::ParamEnv::new(
                 tcx.mk_predicates_from_iter(normalized_preds),
                 param_env.reveal(),
@@ -814,7 +813,7 @@ impl<'tcx> AutoTraitFinder<'tcx> {
 
                     match (evaluate(c1), evaluate(c2)) {
                         (Ok(c1), Ok(c2)) => {
-                            match selcx.infcx.at(&obligation.cause, obligation.param_env).eq(c1, c2)
+                            match selcx.infcx.at(&obligation.cause, obligation.param_env).eq(DefineOpaqueTypes::No,c1, c2)
                             {
                                 Ok(_) => (),
                                 Err(_) => return false,
@@ -830,7 +829,7 @@ impl<'tcx> AutoTraitFinder<'tcx> {
                 // the `ParamEnv`.
                 ty::PredicateKind::WellFormed(..)
                 | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
-                | ty::PredicateKind::AliasEq(..)
+                | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::ObjectSafe(..)
                 | ty::PredicateKind::ClosureKind(..)
                 | ty::PredicateKind::Subtype(..)
@@ -852,23 +851,3 @@ impl<'tcx> AutoTraitFinder<'tcx> {
         infcx.freshen(p)
     }
 }
-
-/// Replaces all ReVars in a type with ty::Region's, using the provided map
-pub struct RegionReplacer<'a, 'tcx> {
-    vid_to_region: &'a FxHashMap<ty::RegionVid, ty::Region<'tcx>>,
-    tcx: TyCtxt<'tcx>,
-}
-
-impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for RegionReplacer<'a, 'tcx> {
-    fn interner(&self) -> TyCtxt<'tcx> {
-        self.tcx
-    }
-
-    fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        (match *r {
-            ty::ReVar(vid) => self.vid_to_region.get(&vid).cloned(),
-            _ => None,
-        })
-        .unwrap_or_else(|| r.super_fold_with(self))
-    }
-}
diff --git a/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs b/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs
index b42a49eb4..28967e1cc 100644
--- a/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs
+++ b/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs
@@ -40,13 +40,16 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> {
         self.obligations.insert(obligation);
     }
 
-    fn collect_remaining_errors(&mut self) -> Vec<FulfillmentError<'tcx>> {
+    fn collect_remaining_errors(
+        &mut self,
+        _infcx: &InferCtxt<'tcx>,
+    ) -> Vec<FulfillmentError<'tcx>> {
         // any remaining obligations are errors
         self.obligations
             .iter()
             .map(|obligation| FulfillmentError {
                 obligation: obligation.clone(),
-                code: FulfillmentErrorCode::CodeAmbiguity,
+                code: FulfillmentErrorCode::CodeAmbiguity { overflow: false },
                 // FIXME - does Chalk have a notation of 'root obligation'?
                 // This is just for diagnostics, so it's okay if this is wrong
                 root_obligation: obligation.clone(),
diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs
index 6b688c322..20c2605f2 100644
--- a/compiler/rustc_trait_selection/src/traits/coherence.rs
+++ b/compiler/rustc_trait_selection/src/traits/coherence.rs
@@ -17,7 +17,7 @@ use crate::traits::{
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::Diagnostic;
 use rustc_hir::def_id::{DefId, CRATE_DEF_ID, LOCAL_CRATE};
-use rustc_infer::infer::{DefiningAnchor, InferCtxt, TyCtxtInferExt};
+use rustc_infer::infer::{DefineOpaqueTypes, DefiningAnchor, InferCtxt, TyCtxtInferExt};
 use rustc_infer::traits::util;
 use rustc_middle::traits::specialization_graph::OverlapMode;
 use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
@@ -75,12 +75,13 @@ pub fn overlapping_impls(
     // Before doing expensive operations like entering an inference context, do
     // a quick check via fast_reject to tell if the impl headers could possibly
     // unify.
-    let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsInfer };
+    let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsCandidateKey };
     let impl1_ref = tcx.impl_trait_ref(impl1_def_id);
     let impl2_ref = tcx.impl_trait_ref(impl2_def_id);
     let may_overlap = match (impl1_ref, impl2_ref) {
-        (Some(a), Some(b)) => iter::zip(a.skip_binder().substs, b.skip_binder().substs)
-            .all(|(arg1, arg2)| drcx.generic_args_may_unify(arg1, arg2)),
+        (Some(a), Some(b)) => {
+            drcx.substs_refs_may_unify(a.skip_binder().substs, b.skip_binder().substs)
+        }
         (None, None) => {
             let self_ty1 = tcx.type_of(impl1_def_id).skip_binder();
             let self_ty2 = tcx.type_of(impl2_def_id).skip_binder();
@@ -95,8 +96,11 @@ pub fn overlapping_impls(
         return None;
     }
 
-    let infcx =
-        tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bubble).intercrate().build();
+    let infcx = tcx
+        .infer_ctxt()
+        .with_opaque_type_inference(DefiningAnchor::Bubble)
+        .intercrate(true)
+        .build();
     let selcx = &mut SelectionContext::new(&infcx);
     let overlaps =
         overlap(selcx, skip_leak_check, impl1_def_id, impl2_def_id, overlap_mode).is_some();
@@ -107,8 +111,11 @@ pub fn overlapping_impls(
     // In the case where we detect an error, run the check again, but
     // this time tracking intercrate ambiguity causes for better
     // diagnostics. (These take time and can lead to false errors.)
-    let infcx =
-        tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bubble).intercrate().build();
+    let infcx = tcx
+        .infer_ctxt()
+        .with_opaque_type_inference(DefiningAnchor::Bubble)
+        .intercrate(true)
+        .build();
     let selcx = &mut SelectionContext::new(&infcx);
     selcx.enable_tracking_intercrate_ambiguity_causes();
     Some(overlap(selcx, skip_leak_check, impl1_def_id, impl2_def_id, overlap_mode).unwrap())
@@ -181,7 +188,7 @@ fn overlap_within_probe<'cx, 'tcx>(
     let impl1_header = with_fresh_ty_vars(selcx, param_env, impl1_def_id);
     let impl2_header = with_fresh_ty_vars(selcx, param_env, impl2_def_id);
 
-    let obligations = equate_impl_headers(selcx, &impl1_header, &impl2_header)?;
+    let obligations = equate_impl_headers(selcx.infcx, &impl1_header, &impl2_header)?;
     debug!("overlap: unification check succeeded");
 
     if overlap_mode.use_implicit_negative() {
@@ -207,20 +214,25 @@ fn overlap_within_probe<'cx, 'tcx>(
     Some(OverlapResult { impl_header, intercrate_ambiguity_causes, involves_placeholder })
 }
 
-fn equate_impl_headers<'cx, 'tcx>(
-    selcx: &mut SelectionContext<'cx, 'tcx>,
-    impl1_header: &ty::ImplHeader<'tcx>,
-    impl2_header: &ty::ImplHeader<'tcx>,
+#[instrument(level = "debug", skip(infcx), ret)]
+fn equate_impl_headers<'tcx>(
+    infcx: &InferCtxt<'tcx>,
+    impl1: &ty::ImplHeader<'tcx>,
+    impl2: &ty::ImplHeader<'tcx>,
 ) -> Option<PredicateObligations<'tcx>> {
-    // Do `a` and `b` unify? If not, no overlap.
-    debug!("equate_impl_headers(impl1_header={:?}, impl2_header={:?}", impl1_header, impl2_header);
-    selcx
-        .infcx
-        .at(&ObligationCause::dummy(), ty::ParamEnv::empty())
-        .define_opaque_types(true)
-        .eq_impl_headers(impl1_header, impl2_header)
-        .map(|infer_ok| infer_ok.obligations)
-        .ok()
+    let result = match (impl1.trait_ref, impl2.trait_ref) {
+        (Some(impl1_ref), Some(impl2_ref)) => infcx
+            .at(&ObligationCause::dummy(), ty::ParamEnv::empty())
+            .eq(DefineOpaqueTypes::Yes, impl1_ref, impl2_ref),
+        (None, None) => infcx.at(&ObligationCause::dummy(), ty::ParamEnv::empty()).eq(
+            DefineOpaqueTypes::Yes,
+            impl1.self_ty,
+            impl2.self_ty,
+        ),
+        _ => bug!("mk_eq_impl_headers given mismatched impl kinds"),
+    };
+
+    result.map(|infer_ok| infer_ok.obligations).ok()
 }
 
 /// Given impl1 and impl2 check if both impls can be satisfied by a common type (including
@@ -294,7 +306,7 @@ fn negative_impl(tcx: TyCtxt<'_>, impl1_def_id: DefId, impl2_def_id: DefId) -> b
         &infcx,
         ObligationCause::dummy(),
         impl_env,
-        tcx.impl_subject(impl1_def_id),
+        tcx.impl_subject(impl1_def_id).subst_identity(),
     ) {
         Ok(s) => s,
         Err(err) => {
@@ -325,7 +337,7 @@ fn equate<'tcx>(
 ) -> bool {
     // do the impls unify? If not, not disjoint.
     let Ok(InferOk { obligations: more_obligations, .. }) =
-        infcx.at(&ObligationCause::dummy(), impl_env).eq(subject1, subject2)
+        infcx.at(&ObligationCause::dummy(), impl_env).eq(DefineOpaqueTypes::No,subject1, subject2)
     else {
         debug!("explicit_disjoint: {:?} does not unify with {:?}", subject1, subject2);
         return true;
@@ -356,8 +368,8 @@ fn negative_impl_exists<'tcx>(
     }
 
     // Try to prove a negative obligation exists for super predicates
-    for o in util::elaborate_predicates(infcx.tcx, iter::once(o.predicate)) {
-        if resolve_negative_obligation(infcx.fork(), &o, body_def_id) {
+    for pred in util::elaborate(infcx.tcx, iter::once(o.predicate)) {
+        if resolve_negative_obligation(infcx.fork(), &o.with(infcx.tcx, pred), body_def_id) {
             return true;
         }
     }
@@ -378,7 +390,10 @@ fn resolve_negative_obligation<'tcx>(
     };
 
     let param_env = o.param_env;
-    if !super::fully_solve_obligation(&infcx, o).is_empty() {
+    let ocx = ObligationCtxt::new(&infcx);
+    ocx.register_obligation(o);
+    let errors = ocx.select_all_or_error();
+    if !errors.is_empty() {
         return false;
     }
 
@@ -388,20 +403,29 @@ fn resolve_negative_obligation<'tcx>(
     let wf_tys = ocx.assumed_wf_types(param_env, DUMMY_SP, body_def_id);
     let outlives_env = OutlivesEnvironment::with_bounds(
         param_env,
-        Some(&infcx),
         infcx.implied_bounds_tys(param_env, body_def_id, wf_tys),
     );
-
-    infcx.process_registered_region_obligations(outlives_env.region_bound_pairs(), param_env);
-
     infcx.resolve_regions(&outlives_env).is_empty()
 }
 
+/// Returns whether all impls which would apply to the `trait_ref`
+/// e.g. `Ty: Trait<Arg>` are already known in the local crate.
+///
+/// This both checks whether any downstream or sibling crates could
+/// implement it and whether an upstream crate can add this impl
+/// without breaking backwards compatibility.
 #[instrument(level = "debug", skip(tcx), ret)]
 pub fn trait_ref_is_knowable<'tcx>(
     tcx: TyCtxt<'tcx>,
     trait_ref: ty::TraitRef<'tcx>,
 ) -> Result<(), Conflict> {
+    if Some(trait_ref.def_id) == tcx.lang_items().fn_ptr_trait() {
+        // The only types implementing `FnPtr` are function pointers,
+        // so if there's no impl of `FnPtr` in the current crate,
+        // then such an impl will never be added in the future.
+        return Ok(());
+    }
+
     if orphan_check_trait_ref(trait_ref, InCrate::Remote).is_ok() {
         // A downstream or cousin crate is allowed to implement some
         // substitution of this trait-ref.
diff --git a/compiler/rustc_trait_selection/src/traits/engine.rs b/compiler/rustc_trait_selection/src/traits/engine.rs
index b20636174..2beebe94b 100644
--- a/compiler/rustc_trait_selection/src/traits/engine.rs
+++ b/compiler/rustc_trait_selection/src/traits/engine.rs
@@ -6,12 +6,14 @@ use super::{ChalkFulfillmentContext, FulfillmentContext};
 use crate::solve::FulfillmentCtxt as NextFulfillmentCtxt;
 use crate::traits::NormalizeExt;
 use rustc_data_structures::fx::FxIndexSet;
+use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_infer::infer::at::ToTrace;
 use rustc_infer::infer::canonical::{
     Canonical, CanonicalQueryResponse, CanonicalVarValues, QueryResponse,
 };
-use rustc_infer::infer::{InferCtxt, InferOk};
+use rustc_infer::infer::outlives::env::OutlivesEnvironment;
+use rustc_infer::infer::{DefineOpaqueTypes, InferCtxt, InferOk};
 use rustc_infer::traits::query::Fallible;
 use rustc_infer::traits::{
     FulfillmentError, Obligation, ObligationCause, PredicateObligation, TraitEngineExt as _,
@@ -128,8 +130,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
     {
         self.infcx
             .at(cause, param_env)
-            .define_opaque_types(true)
-            .eq_exp(a_is_expected, a, b)
+            .eq_exp(DefineOpaqueTypes::Yes, a_is_expected, a, b)
             .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
     }
 
@@ -142,8 +143,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
     ) -> Result<(), TypeError<'tcx>> {
         self.infcx
             .at(cause, param_env)
-            .define_opaque_types(true)
-            .eq(expected, actual)
+            .eq(DefineOpaqueTypes::Yes, expected, actual)
             .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
     }
 
@@ -157,8 +157,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
     ) -> Result<(), TypeError<'tcx>> {
         self.infcx
             .at(cause, param_env)
-            .define_opaque_types(true)
-            .sup(expected, actual)
+            .sub(DefineOpaqueTypes::Yes, expected, actual)
             .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
     }
 
@@ -172,19 +171,37 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
     ) -> Result<(), TypeError<'tcx>> {
         self.infcx
             .at(cause, param_env)
-            .define_opaque_types(true)
-            .sup(expected, actual)
+            .sup(DefineOpaqueTypes::Yes, expected, actual)
             .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
     }
 
+    #[must_use]
     pub fn select_where_possible(&self) -> Vec<FulfillmentError<'tcx>> {
         self.engine.borrow_mut().select_where_possible(self.infcx)
     }
 
+    #[must_use]
     pub fn select_all_or_error(&self) -> Vec<FulfillmentError<'tcx>> {
         self.engine.borrow_mut().select_all_or_error(self.infcx)
     }
 
+    /// Resolves regions and reports errors.
+    ///
+    /// Takes ownership of the context as doing trait solving afterwards
+    /// will result in region constraints getting ignored.
+    pub fn resolve_regions_and_report_errors(
+        self,
+        generic_param_scope: LocalDefId,
+        outlives_env: &OutlivesEnvironment<'tcx>,
+    ) -> Result<(), ErrorGuaranteed> {
+        let errors = self.infcx.resolve_regions(&outlives_env);
+        if errors.is_empty() {
+            Ok(())
+        } else {
+            Err(self.infcx.err_ctxt().report_region_errors(generic_param_scope, &errors))
+        }
+    }
+
     pub fn assumed_wf_types(
         &self,
         param_env: ty::ParamEnv<'tcx>,
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs
index 84045c4d0..0475f24d8 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs
@@ -1,6 +1,6 @@
 use rustc_hir::def_id::DefId;
 use rustc_infer::infer::{InferCtxt, LateBoundRegionConversionTime};
-use rustc_infer::traits::util::elaborate_predicates_with_span;
+use rustc_infer::traits::util::elaborate;
 use rustc_infer::traits::{Obligation, ObligationCause, TraitObligation};
 use rustc_middle::ty;
 use rustc_span::{Span, DUMMY_SP};
@@ -82,15 +82,15 @@ pub fn recompute_applicable_impls<'tcx>(
 
     let predicates =
         tcx.predicates_of(obligation.cause.body_id.to_def_id()).instantiate_identity(tcx);
-    for obligation in elaborate_predicates_with_span(tcx, predicates.into_iter()) {
-        let kind = obligation.predicate.kind();
+    for (pred, span) in elaborate(tcx, predicates.into_iter()) {
+        let kind = pred.kind();
         if let ty::PredicateKind::Clause(ty::Clause::Trait(trait_pred)) = kind.skip_binder()
             && param_env_candidate_may_apply(kind.rebind(trait_pred))
         {
             if kind.rebind(trait_pred.trait_ref) == ty::TraitRef::identity(tcx, trait_pred.def_id()) {
                 ambiguities.push(Ambiguity::ParamEnv(tcx.def_span(trait_pred.def_id())))
             } else {
-                ambiguities.push(Ambiguity::ParamEnv(obligation.cause.span))
+                ambiguities.push(Ambiguity::ParamEnv(span))
             }
         }
     }
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/method_chain.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/method_chain.rs
index 1174efdbf..7e1dba4ed 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/method_chain.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/method_chain.rs
@@ -21,10 +21,6 @@ impl<'a, 'tcx> TypeRelation<'tcx> for CollectAllMismatches<'a, 'tcx> {
         self.infcx.tcx
     }
 
-    fn intercrate(&self) -> bool {
-        false
-    }
-
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
         self.param_env
     }
@@ -33,10 +29,6 @@ impl<'a, 'tcx> TypeRelation<'tcx> for CollectAllMismatches<'a, 'tcx> {
         true
     }
 
-    fn mark_ambiguous(&mut self) {
-        bug!()
-    }
-
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
         _: ty::Variance,
@@ -92,6 +84,11 @@ impl<'a, 'tcx> TypeRelation<'tcx> for CollectAllMismatches<'a, 'tcx> {
 }
 
 impl<'tcx> ObligationEmittingRelation<'tcx> for CollectAllMismatches<'_, 'tcx> {
+    fn alias_relate_direction(&self) -> ty::AliasRelationDirection {
+        // FIXME(deferred_projection_equality): We really should get rid of this relation.
+        ty::AliasRelationDirection::Equate
+    }
+
     fn register_obligations(&mut self, _obligations: PredicateObligations<'tcx>) {
         // FIXME(deferred_projection_equality)
     }
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs
index a844a1494..1b741b730 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs
@@ -24,16 +24,15 @@ use rustc_errors::{
 };
 use rustc_hir as hir;
 use rustc_hir::def::Namespace;
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::Visitor;
-use rustc_hir::GenericParam;
-use rustc_hir::Item;
-use rustc_hir::Node;
+use rustc_hir::{GenericParam, Item, Node};
 use rustc_infer::infer::error_reporting::TypeErrCtxt;
 use rustc_infer::infer::{InferOk, TypeTrace};
 use rustc_middle::traits::select::OverflowError;
 use rustc_middle::ty::abstract_const::NotConstEvaluatable;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
+use rustc_middle::ty::fast_reject::TreatProjections;
 use rustc_middle::ty::fold::{TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::print::{with_forced_trimmed_paths, FmtPrinter, Print};
 use rustc_middle::ty::{
@@ -126,11 +125,9 @@ pub trait TypeErrCtxtExt<'tcx> {
             + Print<'tcx, FmtPrinter<'tcx, 'tcx>, Output = FmtPrinter<'tcx, 'tcx>>,
         <T as Print<'tcx, FmtPrinter<'tcx, 'tcx>>>::Error: std::fmt::Debug;
 
-    fn report_fulfillment_errors(
-        &self,
-        errors: &[FulfillmentError<'tcx>],
-        body_id: Option<hir::BodyId>,
-    ) -> ErrorGuaranteed;
+    fn report_overflow_no_abort(&self, obligation: PredicateObligation<'tcx>) -> ErrorGuaranteed;
+
+    fn report_fulfillment_errors(&self, errors: &[FulfillmentError<'tcx>]) -> ErrorGuaranteed;
 
     fn report_overflow_obligation<T>(
         &self,
@@ -388,11 +385,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
 }
 
 impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
-    fn report_fulfillment_errors(
-        &self,
-        errors: &[FulfillmentError<'tcx>],
-        body_id: Option<hir::BodyId>,
-    ) -> ErrorGuaranteed {
+    fn report_fulfillment_errors(&self, errors: &[FulfillmentError<'tcx>]) -> ErrorGuaranteed {
         #[derive(Debug)]
         struct ErrorDescriptor<'tcx> {
             predicate: ty::Predicate<'tcx>,
@@ -469,7 +462,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         for from_expansion in [false, true] {
             for (error, suppressed) in iter::zip(errors, &is_suppressed) {
                 if !suppressed && error.obligation.cause.span.from_expansion() == from_expansion {
-                    self.report_fulfillment_error(error, body_id);
+                    self.report_fulfillment_error(error);
                 }
             }
         }
@@ -611,6 +604,14 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         );
     }
 
+    fn report_overflow_no_abort(&self, obligation: PredicateObligation<'tcx>) -> ErrorGuaranteed {
+        let obligation = self.resolve_vars_if_possible(obligation);
+        let mut err = self.build_overflow_error(&obligation.predicate, obligation.cause.span, true);
+        self.note_obligation_cause(&mut err, &obligation);
+        self.point_at_returns_when_relevant(&mut err, &obligation);
+        err.emit()
+    }
+
     fn report_selection_error(
         &self,
         mut obligation: PredicateObligation<'tcx>,
@@ -672,6 +673,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             return;
                         }
                         let trait_ref = trait_predicate.to_poly_trait_ref();
+
                         let (post_message, pre_message, type_def) = self
                             .get_parent_trait_ref(obligation.cause.code())
                             .map(|(t, s)| {
@@ -711,33 +713,45 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             (message, note, append_const_msg)
                         };
 
-                        let mut err = struct_span_err!(
-                            self.tcx.sess,
-                            span,
-                            E0277,
-                            "{}",
-                            message
-                                .and_then(|cannot_do_this| {
-                                    match (predicate_is_const, append_const_msg) {
-                                        // do nothing if predicate is not const
-                                        (false, _) => Some(cannot_do_this),
-                                        // suggested using default post message
-                                        (true, Some(None)) => {
-                                            Some(format!("{cannot_do_this} in const contexts"))
-                                        }
-                                        // overridden post message
-                                        (true, Some(Some(post_message))) => {
-                                            Some(format!("{cannot_do_this}{post_message}"))
-                                        }
-                                        // fallback to generic message
-                                        (true, None) => None,
+                        let err_msg = message
+                            .and_then(|cannot_do_this| {
+                                match (predicate_is_const, append_const_msg) {
+                                    // do nothing if predicate is not const
+                                    (false, _) => Some(cannot_do_this),
+                                    // suggested using default post message
+                                    (true, Some(None)) => {
+                                        Some(format!("{cannot_do_this} in const contexts"))
                                     }
-                                })
-                                .unwrap_or_else(|| format!(
+                                    // overridden post message
+                                    (true, Some(Some(post_message))) => {
+                                        Some(format!("{cannot_do_this}{post_message}"))
+                                    }
+                                    // fallback to generic message
+                                    (true, None) => None,
+                                }
+                            })
+                            .unwrap_or_else(|| {
+                                format!(
                                     "the trait bound `{}` is not satisfied{}",
                                     trait_predicate, post_message,
-                                ))
-                        );
+                                )
+                            });
+
+                        let (err_msg, safe_transmute_explanation) = if Some(trait_ref.def_id())
+                            == self.tcx.lang_items().transmute_trait()
+                        {
+                            // Recompute the safe transmute reason and use that for the error reporting
+                            self.get_safe_transmute_error_and_reason(
+                                trait_predicate,
+                                obligation.clone(),
+                                trait_ref,
+                                span,
+                            )
+                        } else {
+                            (err_msg, None)
+                        };
+
+                        let mut err = struct_span_err!(self.tcx.sess, span, E0277, "{}", err_msg);
 
                         if is_try_conversion && let Some(ret_span) = self.return_type_span(&obligation) {
                             err.span_label(
@@ -827,6 +841,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                 // at the type param with a label to suggest constraining it.
                                 err.help(&explanation);
                             }
+                        } else if let Some(custom_explanation) = safe_transmute_explanation {
+                            err.span_label(span, custom_explanation);
                         } else {
                             err.span_label(span, explanation);
                         }
@@ -955,8 +971,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             );
                         }
 
-                        let body_hir_id =
-                            self.tcx.hir().local_def_id_to_hir_id(obligation.cause.body_id);
+                        let body_def_id = obligation.cause.body_id;
                         // Try to report a help message
                         if is_fn_trait
                             && let Ok((implemented_kind, params)) = self.type_implements_fn_trait(
@@ -1035,9 +1050,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             // Can't show anything else useful, try to find similar impls.
                             let impl_candidates = self.find_similar_impl_candidates(trait_predicate);
                             if !self.report_similar_impl_candidates(
-                                impl_candidates,
+                                &impl_candidates,
                                 trait_ref,
-                                body_hir_id,
+                                body_def_id,
                                 &mut err,
                                 true,
                             ) {
@@ -1071,14 +1086,21 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                     let impl_candidates =
                                         self.find_similar_impl_candidates(trait_pred);
                                     self.report_similar_impl_candidates(
-                                        impl_candidates,
+                                        &impl_candidates,
                                         trait_ref,
-                                        body_hir_id,
+                                        body_def_id,
                                         &mut err,
                                         true,
                                     );
                                 }
                             }
+
+                            self.maybe_suggest_convert_to_slice(
+                                &mut err,
+                                trait_ref,
+                                impl_candidates.as_slice(),
+                                span,
+                            );
                         }
 
                         // Changing mutability doesn't make a difference to whether we have
@@ -1279,16 +1301,26 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         "TypeWellFormedFromEnv predicate should only exist in the environment"
                     ),
 
-                    ty::PredicateKind::AliasEq(..) => span_bug!(
+                    ty::PredicateKind::AliasRelate(..) => span_bug!(
                         span,
-                        "AliasEq predicate should never be the predicate cause of a SelectionError"
+                        "AliasRelate predicate should never be the predicate cause of a SelectionError"
                     ),
 
                     ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
-                        self.tcx.sess.struct_span_err(
+                        let mut diag = self.tcx.sess.struct_span_err(
                             span,
                             &format!("the constant `{}` is not of type `{}`", ct, ty),
-                        )
+                        );
+                        self.note_type_err(
+                            &mut diag,
+                            &obligation.cause,
+                            None,
+                            None,
+                            TypeError::Sorts(ty::error::ExpectedFound::new(true, ty, ct.ty())),
+                            false,
+                            false,
+                        );
+                        diag
                     }
                 }
             }
@@ -1494,11 +1526,7 @@ trait InferCtxtPrivExt<'tcx> {
     // `error` occurring implies that `cond` occurs.
     fn error_implies(&self, cond: ty::Predicate<'tcx>, error: ty::Predicate<'tcx>) -> bool;
 
-    fn report_fulfillment_error(
-        &self,
-        error: &FulfillmentError<'tcx>,
-        body_id: Option<hir::BodyId>,
-    );
+    fn report_fulfillment_error(&self, error: &FulfillmentError<'tcx>);
 
     fn report_projection_error(
         &self,
@@ -1529,9 +1557,9 @@ trait InferCtxtPrivExt<'tcx> {
 
     fn report_similar_impl_candidates(
         &self,
-        impl_candidates: Vec<ImplCandidate<'tcx>>,
+        impl_candidates: &[ImplCandidate<'tcx>],
         trait_ref: ty::PolyTraitRef<'tcx>,
-        body_id: hir::HirId,
+        body_def_id: LocalDefId,
         err: &mut Diagnostic,
         other: bool,
     ) -> bool;
@@ -1561,11 +1589,7 @@ trait InferCtxtPrivExt<'tcx> {
         trait_ref_and_ty: ty::Binder<'tcx, (ty::TraitPredicate<'tcx>, Ty<'tcx>)>,
     ) -> PredicateObligation<'tcx>;
 
-    fn maybe_report_ambiguity(
-        &self,
-        obligation: &PredicateObligation<'tcx>,
-        body_id: Option<hir::BodyId>,
-    );
+    fn maybe_report_ambiguity(&self, obligation: &PredicateObligation<'tcx>);
 
     fn predicate_can_apply(
         &self,
@@ -1602,6 +1626,14 @@ trait InferCtxtPrivExt<'tcx> {
         obligated_types: &mut Vec<Ty<'tcx>>,
         cause_code: &ObligationCauseCode<'tcx>,
     ) -> bool;
+
+    fn get_safe_transmute_error_and_reason(
+        &self,
+        trait_predicate: ty::Binder<'tcx, ty::TraitPredicate<'tcx>>,
+        obligation: Obligation<'tcx, ty::Predicate<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
+        span: Span,
+    ) -> (String, Option<String>);
 }
 
 impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
@@ -1625,8 +1657,8 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             }
         };
 
-        for obligation in super::elaborate_predicates(self.tcx, std::iter::once(cond)) {
-            let bound_predicate = obligation.predicate.kind();
+        for pred in super::elaborate(self.tcx, std::iter::once(cond)) {
+            let bound_predicate = pred.kind();
             if let ty::PredicateKind::Clause(ty::Clause::Trait(implication)) =
                 bound_predicate.skip_binder()
             {
@@ -1647,11 +1679,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     }
 
     #[instrument(skip(self), level = "debug")]
-    fn report_fulfillment_error(
-        &self,
-        error: &FulfillmentError<'tcx>,
-        body_id: Option<hir::BodyId>,
-    ) {
+    fn report_fulfillment_error(&self, error: &FulfillmentError<'tcx>) {
         match error.code {
             FulfillmentErrorCode::CodeSelectionError(ref selection_error) => {
                 self.report_selection_error(
@@ -1663,8 +1691,11 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             FulfillmentErrorCode::CodeProjectionError(ref e) => {
                 self.report_projection_error(&error.obligation, e);
             }
-            FulfillmentErrorCode::CodeAmbiguity => {
-                self.maybe_report_ambiguity(&error.obligation, body_id);
+            FulfillmentErrorCode::CodeAmbiguity { overflow: false } => {
+                self.maybe_report_ambiguity(&error.obligation);
+            }
+            FulfillmentErrorCode::CodeAmbiguity { overflow: true } => {
+                self.report_overflow_no_abort(error.obligation.clone());
             }
             FulfillmentErrorCode::CodeSubtypeError(ref expected_found, ref err) => {
                 self.report_mismatched_types(
@@ -1768,7 +1799,10 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
                 // constrain inference variables a bit more to nested obligations from normalize so
                 // we can have more helpful errors.
-                ocx.select_where_possible();
+                //
+                // we intentionally drop errors from normalization here,
+                // since the normalization is just done to improve the error message.
+                let _ = ocx.select_where_possible();
 
                 if let Err(new_err) = ocx.eq_exp(
                     &obligation.cause,
@@ -1815,12 +1849,17 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     })
                     .and_then(|(trait_assoc_item, id)| {
                         let trait_assoc_ident = trait_assoc_item.ident(self.tcx);
-                        self.tcx.find_map_relevant_impl(id, proj.projection_ty.self_ty(), |did| {
-                            self.tcx
-                                .associated_items(did)
-                                .in_definition_order()
-                                .find(|assoc| assoc.ident(self.tcx) == trait_assoc_ident)
-                        })
+                        self.tcx.find_map_relevant_impl(
+                            id,
+                            proj.projection_ty.self_ty(),
+                            TreatProjections::ForLookup,
+                            |did| {
+                                self.tcx
+                                    .associated_items(did)
+                                    .in_definition_order()
+                                    .find(|assoc| assoc.ident(self.tcx) == trait_assoc_ident)
+                            },
+                        )
                     })
                     .and_then(|item| match self.tcx.hir().get_if_local(item.def_id) {
                         Some(
@@ -2027,9 +2066,9 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
     fn report_similar_impl_candidates(
         &self,
-        impl_candidates: Vec<ImplCandidate<'tcx>>,
+        impl_candidates: &[ImplCandidate<'tcx>],
         trait_ref: ty::PolyTraitRef<'tcx>,
-        body_id: hir::HirId,
+        body_def_id: LocalDefId,
         err: &mut Diagnostic,
         other: bool,
     ) -> bool {
@@ -2120,9 +2159,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         // FIXME(compiler-errors): This could be generalized, both to
                         // be more granular, and probably look past other `#[fundamental]`
                         // types, too.
-                        self.tcx
-                            .visibility(def.did())
-                            .is_accessible_from(body_id.owner.def_id, self.tcx)
+                        self.tcx.visibility(def.did()).is_accessible_from(body_def_id, self.tcx)
                     } else {
                         true
                     }
@@ -2138,7 +2175,8 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         // Prefer more similar candidates first, then sort lexicographically
         // by their normalized string representation.
         let mut normalized_impl_candidates_and_similarities = impl_candidates
-            .into_iter()
+            .iter()
+            .copied()
             .map(|ImplCandidate { trait_ref, similarity }| {
                 // FIXME(compiler-errors): This should be using `NormalizeExt::normalize`
                 let normalized = self
@@ -2193,7 +2231,12 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         trait_ref: &ty::PolyTraitRef<'tcx>,
     ) -> bool {
         let get_trait_impl = |trait_def_id| {
-            self.tcx.find_map_relevant_impl(trait_def_id, trait_ref.skip_binder().self_ty(), Some)
+            self.tcx.find_map_relevant_impl(
+                trait_def_id,
+                trait_ref.skip_binder().self_ty(),
+                TreatProjections::ForLookup,
+                Some,
+            )
         };
         let required_trait_path = self.tcx.def_path_str(trait_ref.def_id());
         let traits_with_same_path: std::collections::BTreeSet<_> = self
@@ -2231,11 +2274,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     }
 
     #[instrument(skip(self), level = "debug")]
-    fn maybe_report_ambiguity(
-        &self,
-        obligation: &PredicateObligation<'tcx>,
-        body_id: Option<hir::BodyId>,
-    ) {
+    fn maybe_report_ambiguity(&self, obligation: &PredicateObligation<'tcx>) {
         // Unable to successfully determine, probably means
         // insufficient type information, but could mean
         // ambiguous impls. The latter *ought* to be a
@@ -2277,7 +2316,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 if self.tcx.lang_items().sized_trait() == Some(trait_ref.def_id()) {
                     if let None = self.tainted_by_errors() {
                         self.emit_inference_failure_err(
-                            body_id,
+                            obligation.cause.body_id,
                             span,
                             trait_ref.self_ty().skip_binder().into(),
                             ErrorCode::E0282,
@@ -2304,7 +2343,13 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 let subst = data.trait_ref.substs.iter().find(|s| s.has_non_region_infer());
 
                 let mut err = if let Some(subst) = subst {
-                    self.emit_inference_failure_err(body_id, span, subst, ErrorCode::E0283, true)
+                    self.emit_inference_failure_err(
+                        obligation.cause.body_id,
+                        span,
+                        subst,
+                        ErrorCode::E0283,
+                        true,
+                    )
                 } else {
                     struct_span_err!(
                         self.tcx.sess,
@@ -2348,12 +2393,10 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                 predicate.to_opt_poly_trait_pred().unwrap(),
                             );
                             if impl_candidates.len() < 10 {
-                                let hir =
-                                    self.tcx.hir().local_def_id_to_hir_id(obligation.cause.body_id);
                                 self.report_similar_impl_candidates(
-                                    impl_candidates,
+                                    impl_candidates.as_slice(),
                                     trait_ref,
-                                    body_id.map(|id| id.hir_id).unwrap_or(hir),
+                                    obligation.cause.body_id,
                                     &mut err,
                                     false,
                                 );
@@ -2375,8 +2418,8 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     self.suggest_fully_qualified_path(&mut err, def_id, span, trait_ref.def_id());
                 }
 
-                if let (Some(body_id), Some(ty::subst::GenericArgKind::Type(_))) =
-                    (body_id, subst.map(|subst| subst.unpack()))
+                if let Some(ty::subst::GenericArgKind::Type(_)) = subst.map(|subst| subst.unpack())
+                    && let Some(body_id) = self.tcx.hir().maybe_body_owned_by(obligation.cause.body_id)
                 {
                     let mut expr_finder = FindExprBySpan::new(span);
                     expr_finder.visit_expr(&self.tcx.hir().body(body_id).value);
@@ -2473,7 +2516,13 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     return;
                 }
 
-                self.emit_inference_failure_err(body_id, span, arg, ErrorCode::E0282, false)
+                self.emit_inference_failure_err(
+                    obligation.cause.body_id,
+                    span,
+                    arg,
+                    ErrorCode::E0282,
+                    false,
+                )
             }
 
             ty::PredicateKind::Subtype(data) => {
@@ -2487,7 +2536,13 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 let SubtypePredicate { a_is_expected: _, a, b } = data;
                 // both must be type variables, or the other would've been instantiated
                 assert!(a.is_ty_var() && b.is_ty_var());
-                self.emit_inference_failure_err(body_id, span, a.into(), ErrorCode::E0282, true)
+                self.emit_inference_failure_err(
+                    obligation.cause.body_id,
+                    span,
+                    a.into(),
+                    ErrorCode::E0282,
+                    true,
+                )
             }
             ty::PredicateKind::Clause(ty::Clause::Projection(data)) => {
                 if predicate.references_error() || self.tainted_by_errors().is_some() {
@@ -2501,7 +2556,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     .find(|g| g.has_non_region_infer());
                 if let Some(subst) = subst {
                     let mut err = self.emit_inference_failure_err(
-                        body_id,
+                        obligation.cause.body_id,
                         span,
                         subst,
                         ErrorCode::E0284,
@@ -2530,7 +2585,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 let subst = data.walk().find(|g| g.is_non_region_infer());
                 if let Some(subst) = subst {
                     let err = self.emit_inference_failure_err(
-                        body_id,
+                        obligation.cause.body_id,
                         span,
                         subst,
                         ErrorCode::E0284,
@@ -2863,6 +2918,63 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         }
         false
     }
+
+    fn get_safe_transmute_error_and_reason(
+        &self,
+        trait_predicate: ty::Binder<'tcx, ty::TraitPredicate<'tcx>>,
+        obligation: Obligation<'tcx, ty::Predicate<'tcx>>,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
+        span: Span,
+    ) -> (String, Option<String>) {
+        let src_and_dst = trait_predicate.map_bound(|p| rustc_transmute::Types {
+            dst: p.trait_ref.substs.type_at(0),
+            src: p.trait_ref.substs.type_at(1),
+        });
+        let scope = trait_ref.skip_binder().substs.type_at(2);
+        let Some(assume) =
+            rustc_transmute::Assume::from_const(self.infcx.tcx, obligation.param_env, trait_ref.skip_binder().substs.const_at(3)) else {
+                span_bug!(span, "Unable to construct rustc_transmute::Assume where it was previously possible");
+            };
+        match rustc_transmute::TransmuteTypeEnv::new(self.infcx).is_transmutable(
+            obligation.cause,
+            src_and_dst,
+            scope,
+            assume,
+        ) {
+            rustc_transmute::Answer::No(reason) => {
+                let dst = trait_ref.skip_binder().substs.type_at(0);
+                let src = trait_ref.skip_binder().substs.type_at(1);
+                let custom_err_msg = format!("`{src}` cannot be safely transmuted into `{dst}` in the defining scope of `{scope}`").to_string();
+                let reason_msg = match reason {
+                    rustc_transmute::Reason::SrcIsUnspecified => {
+                        format!("`{src}` does not have a well-specified layout").to_string()
+                    }
+                    rustc_transmute::Reason::DstIsUnspecified => {
+                        format!("`{dst}` does not have a well-specified layout").to_string()
+                    }
+                    rustc_transmute::Reason::DstIsBitIncompatible => {
+                        format!("At least one value of `{src}` isn't a bit-valid value of `{dst}`")
+                            .to_string()
+                    }
+                    rustc_transmute::Reason::DstIsPrivate => format!(
+                        "`{dst}` is or contains a type or field that is not visible in that scope"
+                    )
+                    .to_string(),
+                    // FIXME(bryangarza): Include the number of bytes of src and dst
+                    rustc_transmute::Reason::DstIsTooBig => {
+                        format!("The size of `{src}` is smaller than the size of `{dst}`")
+                    }
+                };
+                (custom_err_msg, Some(reason_msg))
+            }
+            // Should never get a Yes at this point! We already ran it before, and did not get a Yes.
+            rustc_transmute::Answer::Yes => span_bug!(
+                span,
+                "Inconsistent rustc_transmute::is_transmutable(...) result, got Yes",
+            ),
+            _ => span_bug!(span, "Unsupported rustc_transmute::Reason variant"),
+        }
+    }
 }
 
 /// Crude way of getting back an `Expr` from a `Span`.
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
index b3bf9ad59..a9c4e1268 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
@@ -149,9 +149,17 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             .unwrap_or_else(|| (trait_ref.def_id(), trait_ref.skip_binder().substs));
         let trait_ref = trait_ref.skip_binder();
 
-        let body_hir = self.tcx.hir().local_def_id_to_hir_id(obligation.cause.body_id);
-        let mut flags =
-            vec![(sym::ItemContext, self.describe_enclosure(body_hir).map(|s| s.to_owned()))];
+        let mut flags = vec![];
+        // FIXME(-Zlower-impl-trait-in-trait-to-assoc-ty): HIR is not present for RPITITs,
+        // but I guess we could synthesize one here. We don't see any errors that rely on
+        // that yet, though.
+        let enclosure =
+            if let Some(body_hir) = self.tcx.opt_local_def_id_to_hir_id(obligation.cause.body_id) {
+                self.describe_enclosure(body_hir).map(|s| s.to_owned())
+            } else {
+                None
+            };
+        flags.push((sym::ItemContext, enclosure));
 
         match obligation.cause.code() {
             ObligationCauseCode::BuiltinDerivedObligation(..)
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
index 66d74fd05..fb75ec767 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
@@ -1,7 +1,7 @@
 // ignore-tidy-filelength
 
 use super::{
-    DefIdOrName, FindExprBySpan, Obligation, ObligationCause, ObligationCauseCode,
+    DefIdOrName, FindExprBySpan, ImplCandidate, Obligation, ObligationCause, ObligationCauseCode,
     PredicateObligation,
 };
 
@@ -30,7 +30,7 @@ use rustc_middle::hir::map;
 use rustc_middle::ty::error::TypeError::{self, Sorts};
 use rustc_middle::ty::relate::TypeRelation;
 use rustc_middle::ty::{
-    self, suggest_arbitrary_trait_bound, suggest_constraining_type_param, AdtKind, DefIdTree,
+    self, suggest_arbitrary_trait_bound, suggest_constraining_type_param, AdtKind,
     GeneratorDiagnosticData, GeneratorInteriorTypeCause, Infer, InferTy, InternalSubsts,
     IsSuggestable, ToPredicate, Ty, TyCtxt, TypeAndMut, TypeFoldable, TypeFolder,
     TypeSuperFoldable, TypeVisitableExt, TypeckResults,
@@ -212,7 +212,7 @@ pub trait TypeErrCtxtExt<'tcx> {
 
     fn extract_callable_info(
         &self,
-        hir_id: HirId,
+        body_id: LocalDefId,
         param_env: ty::ParamEnv<'tcx>,
         found: Ty<'tcx>,
     ) -> Option<(DefIdOrName, Ty<'tcx>, Vec<Ty<'tcx>>)>;
@@ -382,6 +382,14 @@ pub trait TypeErrCtxtExt<'tcx> {
         body_id: hir::HirId,
         param_env: ty::ParamEnv<'tcx>,
     ) -> Vec<Option<(Span, (DefId, Ty<'tcx>))>>;
+
+    fn maybe_suggest_convert_to_slice(
+        &self,
+        err: &mut Diagnostic,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
+        candidate_impls: &[ImplCandidate<'tcx>],
+        span: Span,
+    );
 }
 
 fn predicate_constraint(generics: &hir::Generics<'_>, pred: ty::Predicate<'_>) -> (Span, String) {
@@ -412,6 +420,7 @@ fn suggest_restriction<'tcx>(
 ) {
     if hir_generics.where_clause_span.from_expansion()
         || hir_generics.where_clause_span.desugaring_kind().is_some()
+        || projection.map_or(false, |projection| tcx.opt_rpitit_info(projection.def_id).is_some())
     {
         return;
     }
@@ -900,9 +909,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             trait_pred.self_ty(),
         );
 
-        let body_hir_id = self.tcx.hir().local_def_id_to_hir_id(obligation.cause.body_id);
         let Some((def_id_or_name, output, inputs)) = self.extract_callable_info(
-            body_hir_id,
+            obligation.cause.body_id,
             obligation.param_env,
             self_ty,
         ) else { return false; };
@@ -1104,10 +1112,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     /// Extracts information about a callable type for diagnostics. This is a
     /// heuristic -- it doesn't necessarily mean that a type is always callable,
     /// because the callable type must also be well-formed to be called.
-    // FIXME(vincenzopalazzo): move the HirId to a LocalDefId
     fn extract_callable_info(
         &self,
-        hir_id: HirId,
+        body_id: LocalDefId,
         param_env: ty::ParamEnv<'tcx>,
         found: Ty<'tcx>,
     ) -> Option<(DefIdOrName, Ty<'tcx>, Vec<Ty<'tcx>>)> {
@@ -1159,7 +1166,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     })
                 }
                 ty::Param(param) => {
-                    let generics = self.tcx.generics_of(hir_id.owner.to_def_id());
+                    let generics = self.tcx.generics_of(body_id);
                     let name = if generics.count() > param.index as usize
                         && let def = generics.param_at(param.index as usize, self.tcx)
                         && matches!(def.kind, ty::GenericParamDefKind::Type { .. })
@@ -1349,6 +1356,31 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             Applicability::MaybeIncorrect,
                         );
                     } else {
+                        let is_mut = mut_ref_self_ty_satisfies_pred || ref_inner_ty_mut;
+                        let sugg_prefix = format!("&{}", if is_mut { "mut " } else { "" });
+                        let sugg_msg = &format!(
+                            "consider{} borrowing here",
+                            if is_mut { " mutably" } else { "" }
+                        );
+
+                        // Issue #109436, we need to add parentheses properly for method calls
+                        // for example, `foo.into()` should be `(&foo).into()`
+                        if let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(
+                            self.tcx.sess.source_map().span_look_ahead(span, Some("."), Some(50)),
+                        ) {
+                            if snippet == "." {
+                                err.multipart_suggestion_verbose(
+                                    sugg_msg,
+                                    vec![
+                                        (span.shrink_to_lo(), format!("({}", sugg_prefix)),
+                                        (span.shrink_to_hi(), ")".to_string()),
+                                    ],
+                                    Applicability::MaybeIncorrect,
+                                );
+                                return true;
+                            }
+                        }
+
                         // Issue #104961, we need to add parentheses properly for compond expressions
                         // for example, `x.starts_with("hi".to_string() + "you")`
                         // should be `x.starts_with(&("hi".to_string() + "you"))`
@@ -1365,14 +1397,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             _ => false,
                         };
 
-                        let is_mut = mut_ref_self_ty_satisfies_pred || ref_inner_ty_mut;
                         let span = if needs_parens { span } else { span.shrink_to_lo() };
-                        let sugg_prefix = format!("&{}", if is_mut { "mut " } else { "" });
-                        let sugg_msg = &format!(
-                            "consider{} borrowing here",
-                            if is_mut { " mutably" } else { "" }
-                        );
-
                         let suggestions = if !needs_parens {
                             vec![(span.shrink_to_lo(), format!("{}", sugg_prefix))]
                         } else {
@@ -2220,7 +2245,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         // - `BuiltinDerivedObligation` with a generator witness (A)
         // - `BuiltinDerivedObligation` with a generator (A)
         // - `BuiltinDerivedObligation` with `impl std::future::Future` (A)
-        // - `BindingObligation` with `impl_send (Send requirement)
+        // - `BindingObligation` with `impl_send` (Send requirement)
         //
         // The first obligation in the chain is the most useful and has the generator that captured
         // the type. The last generator (`outer_generator` below) has information about where the
@@ -2936,9 +2961,6 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             ObligationCauseCode::SizedYieldType => {
                 err.note("the yield type of a generator must have a statically known size");
             }
-            ObligationCauseCode::SizedBoxType => {
-                err.note("the type of a box expression must have a statically known size");
-            }
             ObligationCauseCode::AssignmentLhsSized => {
                 err.note("the left-hand-side of an assignment must have a statically known size");
             }
@@ -3025,8 +3047,6 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     }
                 };
 
-                let identity_future = tcx.require_lang_item(LangItem::IdentityFuture, None);
-
                 // Don't print the tuple of capture types
                 'print: {
                     if !is_upvar_tys_infer_tuple {
@@ -3039,12 +3059,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                 None => err.note(&msg),
                             },
                             ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) => {
-                                // Avoid printing the future from `core::future::identity_future`, it's not helpful
-                                if tcx.parent(*def_id) == identity_future {
-                                    break 'print;
-                                }
-
-                                // If the previous type is `identity_future`, this is the future generated by the body of an async function.
+                                // If the previous type is async fn, this is the future generated by the body of an async function.
                                 // Avoid printing it twice (it was already printed in the `ty::Generator` arm below).
                                 let is_future = tcx.ty_is_opaque_future(ty);
                                 debug!(
@@ -3826,6 +3841,72 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         }
         assocs_in_this_method
     }
+
+    /// If the type that failed selection is an array or a reference to an array,
+    /// but the trait is implemented for slices, suggest that the user converts
+    /// the array into a slice.
+    fn maybe_suggest_convert_to_slice(
+        &self,
+        err: &mut Diagnostic,
+        trait_ref: ty::Binder<'tcx, ty::TraitRef<'tcx>>,
+        candidate_impls: &[ImplCandidate<'tcx>],
+        span: Span,
+    ) {
+        // Three cases where we can make a suggestion:
+        // 1. `[T; _]` (array of T)
+        // 2. `&[T; _]` (reference to array of T)
+        // 3. `&mut [T; _]` (mutable reference to array of T)
+        let (element_ty, mut mutability) = match *trait_ref.skip_binder().self_ty().kind() {
+            ty::Array(element_ty, _) => (element_ty, None),
+
+            ty::Ref(_, pointee_ty, mutability) => match *pointee_ty.kind() {
+                ty::Array(element_ty, _) => (element_ty, Some(mutability)),
+                _ => return,
+            },
+
+            _ => return,
+        };
+
+        // Go through all the candidate impls to see if any of them is for
+        // slices of `element_ty` with `mutability`.
+        let mut is_slice = |candidate: Ty<'tcx>| match *candidate.kind() {
+            ty::RawPtr(ty::TypeAndMut { ty: t, mutbl: m }) | ty::Ref(_, t, m) => {
+                if matches!(*t.kind(), ty::Slice(e) if e == element_ty)
+                    && m == mutability.unwrap_or(m)
+                {
+                    // Use the candidate's mutability going forward.
+                    mutability = Some(m);
+                    true
+                } else {
+                    false
+                }
+            }
+            _ => false,
+        };
+
+        // Grab the first candidate that matches, if any, and make a suggestion.
+        if let Some(slice_ty) = candidate_impls
+            .iter()
+            .map(|trait_ref| trait_ref.trait_ref.self_ty())
+            .find(|t| is_slice(*t))
+        {
+            let msg = &format!("convert the array to a `{}` slice instead", slice_ty);
+
+            if let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(span) {
+                let mut suggestions = vec![];
+                if snippet.starts_with('&') {
+                } else if let Some(hir::Mutability::Mut) = mutability {
+                    suggestions.push((span.shrink_to_lo(), "&mut ".into()));
+                } else {
+                    suggestions.push((span.shrink_to_lo(), "&".into()));
+                }
+                suggestions.push((span.shrink_to_hi(), "[..]".into()));
+                err.multipart_suggestion_verbose(msg, suggestions, Applicability::MaybeIncorrect);
+            } else {
+                err.span_help(span, msg);
+            }
+        }
+    }
 }
 
 /// Add a hint to add a missing borrow or remove an unnecessary one.
@@ -3854,7 +3935,7 @@ fn hint_missing_borrow<'tcx>(
     // This could be a variant constructor, for example.
     let Some(fn_decl) = found_node.fn_decl() else { return; };
 
-    let args = fn_decl.inputs.iter().map(|ty| ty);
+    let args = fn_decl.inputs.iter();
 
     fn get_deref_type_and_refs(mut ty: Ty<'_>) -> (Ty<'_>, Vec<hir::Mutability>) {
         let mut refs = vec![];
diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs
index 944436ab8..26cadab3e 100644
--- a/compiler/rustc_trait_selection/src/traits/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs
@@ -2,6 +2,7 @@ use crate::infer::{InferCtxt, TyOrConstInferVar};
 use rustc_data_structures::obligation_forest::ProcessResult;
 use rustc_data_structures::obligation_forest::{Error, ForestObligation, Outcome};
 use rustc_data_structures::obligation_forest::{ObligationForest, ObligationProcessor};
+use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::traits::ProjectionCacheKey;
 use rustc_infer::traits::{SelectionError, TraitEngine, TraitObligation};
 use rustc_middle::mir::interpret::ErrorHandled;
@@ -132,8 +133,15 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> {
             .register_obligation(PendingPredicateObligation { obligation, stalled_on: vec![] });
     }
 
-    fn collect_remaining_errors(&mut self) -> Vec<FulfillmentError<'tcx>> {
-        self.predicates.to_errors(CodeAmbiguity).into_iter().map(to_fulfillment_error).collect()
+    fn collect_remaining_errors(
+        &mut self,
+        _infcx: &InferCtxt<'tcx>,
+    ) -> Vec<FulfillmentError<'tcx>> {
+        self.predicates
+            .to_errors(CodeAmbiguity { overflow: false })
+            .into_iter()
+            .map(to_fulfillment_error)
+            .collect()
     }
 
     fn select_where_possible(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>> {
@@ -210,6 +218,29 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
     type Error = FulfillmentErrorCode<'tcx>;
     type OUT = Outcome<Self::Obligation, Self::Error>;
 
+    /// Compared to `needs_process_obligation` this and its callees
+    /// contain some optimizations that come at the price of false negatives.
+    ///
+    /// They
+    /// - reduce branching by covering only the most common case
+    /// - take a read-only view of the unification tables which allows skipping undo_log
+    ///   construction.
+    /// - bail out on value-cache misses in ena to avoid pointer chasing
+    /// - hoist RefCell locking out of the loop
+    #[inline]
+    fn skippable_obligations<'b>(
+        &'b self,
+        it: impl Iterator<Item = &'b Self::Obligation>,
+    ) -> usize {
+        let is_unchanged = self.selcx.infcx.is_ty_infer_var_definitely_unchanged();
+
+        it.take_while(|o| match o.stalled_on.as_slice() {
+            [o] => is_unchanged(*o),
+            _ => false,
+        })
+        .count()
+    }
+
     /// Identifies whether a predicate obligation needs processing.
     ///
     /// This is always inlined because it has a single callsite and it is
@@ -337,8 +368,8 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                 ty::PredicateKind::TypeWellFormedFromEnv(..) => {
                     bug!("TypeWellFormedFromEnv is only used for Chalk")
                 }
-                ty::PredicateKind::AliasEq(..) => {
-                    bug!("AliasEq is only used for new solver")
+                ty::PredicateKind::AliasRelate(..) => {
+                    bug!("AliasRelate is only used for new solver")
                 }
             },
             Some(pred) => match pred {
@@ -515,7 +546,7 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                                 if let Ok(new_obligations) = infcx
                                     .at(&obligation.cause, obligation.param_env)
                                     .trace(c1, c2)
-                                    .eq(a.substs, b.substs)
+                                    .eq(DefineOpaqueTypes::No, a.substs, b.substs)
                                 {
                                     return ProcessResult::Changed(mk_pending(
                                         new_obligations.into_obligations(),
@@ -524,8 +555,9 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                             }
                             (_, Unevaluated(_)) | (Unevaluated(_), _) => (),
                             (_, _) => {
-                                if let Ok(new_obligations) =
-                                    infcx.at(&obligation.cause, obligation.param_env).eq(c1, c2)
+                                if let Ok(new_obligations) = infcx
+                                    .at(&obligation.cause, obligation.param_env)
+                                    .eq(DefineOpaqueTypes::No, c1, c2)
                                 {
                                     return ProcessResult::Changed(mk_pending(
                                         new_obligations.into_obligations(),
@@ -565,12 +597,11 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
 
                     match (evaluate(c1), evaluate(c2)) {
                         (Ok(c1), Ok(c2)) => {
-                            match self
-                                .selcx
-                                .infcx
-                                .at(&obligation.cause, obligation.param_env)
-                                .eq(c1, c2)
-                            {
+                            match self.selcx.infcx.at(&obligation.cause, obligation.param_env).eq(
+                                DefineOpaqueTypes::No,
+                                c1,
+                                c2,
+                            ) {
                                 Ok(inf_ok) => {
                                     ProcessResult::Changed(mk_pending(inf_ok.into_obligations()))
                                 }
@@ -606,16 +637,15 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                 ty::PredicateKind::TypeWellFormedFromEnv(..) => {
                     bug!("TypeWellFormedFromEnv is only used for Chalk")
                 }
-                ty::PredicateKind::AliasEq(..) => {
-                    bug!("AliasEq is only used for new solver")
+                ty::PredicateKind::AliasRelate(..) => {
+                    bug!("AliasRelate is only used for new solver")
                 }
                 ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
-                    match self
-                        .selcx
-                        .infcx
-                        .at(&obligation.cause, obligation.param_env)
-                        .eq(ct.ty(), ty)
-                    {
+                    match self.selcx.infcx.at(&obligation.cause, obligation.param_env).eq(
+                        DefineOpaqueTypes::No,
+                        ct.ty(),
+                        ty,
+                    ) {
                         Ok(inf_ok) => ProcessResult::Changed(mk_pending(inf_ok.into_obligations())),
                         Err(_) => ProcessResult::Error(FulfillmentErrorCode::CodeSelectionError(
                             SelectionError::Unimplemented,
diff --git a/compiler/rustc_trait_selection/src/traits/misc.rs b/compiler/rustc_trait_selection/src/traits/misc.rs
index b94346b09..af567c074 100644
--- a/compiler/rustc_trait_selection/src/traits/misc.rs
+++ b/compiler/rustc_trait_selection/src/traits/misc.rs
@@ -87,7 +87,12 @@ pub fn type_allowed_to_implement_copy<'tcx>(
             };
             let ty = ocx.normalize(&normalization_cause, param_env, unnormalized_ty);
             let normalization_errors = ocx.select_where_possible();
-            if !normalization_errors.is_empty() {
+
+            // NOTE: The post-normalization type may also reference errors,
+            // such as when we project to a missing type or we have a mismatch
+            // between expected and found const-generic types. Don't report an
+            // additional copy error here, since it's not typically useful.
+            if !normalization_errors.is_empty() || ty.references_error() {
                 tcx.sess.delay_span_bug(field_span, format!("couldn't normalize struct field `{unnormalized_ty}` when checking Copy implementation"));
                 continue;
             }
@@ -106,17 +111,12 @@ pub fn type_allowed_to_implement_copy<'tcx>(
             // Check regions assuming the self type of the impl is WF
             let outlives_env = OutlivesEnvironment::with_bounds(
                 param_env,
-                Some(&infcx),
                 infcx.implied_bounds_tys(
                     param_env,
                     parent_cause.body_id,
                     FxIndexSet::from_iter([self_type]),
                 ),
             );
-            infcx.process_registered_region_obligations(
-                outlives_env.region_bound_pairs(),
-                param_env,
-            );
             let errors = infcx.resolve_regions(&outlives_env);
             if !errors.is_empty() {
                 infringing.push((field, ty, InfringingFieldsReason::Regions(errors)));
diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs
index 4e30108be..8a203dec8 100644
--- a/compiler/rustc_trait_selection/src/traits/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/mod.rs
@@ -4,7 +4,7 @@
 
 pub mod auto_trait;
 mod chalk_fulfill;
-mod coherence;
+pub(crate) mod coherence;
 pub mod const_evaluatable;
 mod engine;
 pub mod error_reporting;
@@ -28,9 +28,9 @@ use crate::traits::query::evaluate_obligation::InferCtxtExt as _;
 use rustc_errors::ErrorGuaranteed;
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
-use rustc_middle::ty::{self, DefIdTree, ToPredicate, Ty, TyCtxt, TypeSuperVisitable};
+use rustc_middle::ty::{self, ToPredicate, Ty, TyCtxt, TypeSuperVisitable};
 use rustc_middle::ty::{InternalSubsts, SubstsRef};
-use rustc_span::def_id::{DefId, CRATE_DEF_ID};
+use rustc_span::def_id::DefId;
 use rustc_span::Span;
 
 use std::fmt::Debug;
@@ -58,17 +58,12 @@ pub use self::specialize::{specialization_graph, translate_substs, OverlapError}
 pub use self::structural_match::{
     search_for_adt_const_param_violation, search_for_structural_match_violation,
 };
-pub use self::util::{
-    elaborate_obligations, elaborate_predicates, elaborate_predicates_with_span,
-    elaborate_trait_ref, elaborate_trait_refs,
-};
+pub use self::util::elaborate;
 pub use self::util::{expand_trait_aliases, TraitAliasExpander};
-pub use self::util::{
-    get_vtable_index_of_object_method, impl_item_is_final, predicate_for_trait_def, upcast_choices,
-};
+pub use self::util::{get_vtable_index_of_object_method, impl_item_is_final, upcast_choices};
 pub use self::util::{
     supertrait_def_ids, supertraits, transitive_bounds, transitive_bounds_that_define_assoc_type,
-    SupertraitDefIds, Supertraits,
+    SupertraitDefIds,
 };
 
 pub use self::chalk_fulfill::FulfillmentContext as ChalkFulfillmentContext;
@@ -131,29 +126,23 @@ pub fn type_known_to_meet_bound_modulo_regions<'tcx>(
     param_env: ty::ParamEnv<'tcx>,
     ty: Ty<'tcx>,
     def_id: DefId,
-    span: Span,
 ) -> bool {
     let trait_ref = ty::Binder::dummy(infcx.tcx.mk_trait_ref(def_id, [ty]));
-    pred_known_to_hold_modulo_regions(infcx, param_env, trait_ref.without_const(), span)
+    pred_known_to_hold_modulo_regions(infcx, param_env, trait_ref.without_const())
 }
 
-#[instrument(level = "debug", skip(infcx, param_env, span, pred), ret)]
+/// FIXME(@lcnr): this function doesn't seem right and shouldn't exist?
+///
+/// Ping me on zulip if you want to use this method and need help with finding
+/// an appropriate replacement.
+#[instrument(level = "debug", skip(infcx, param_env, pred), ret)]
 fn pred_known_to_hold_modulo_regions<'tcx>(
     infcx: &InferCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     pred: impl ToPredicate<'tcx> + TypeVisitable<TyCtxt<'tcx>>,
-    span: Span,
 ) -> bool {
     let has_non_region_infer = pred.has_non_region_infer();
-    let obligation = Obligation {
-        param_env,
-        // We can use a dummy node-id here because we won't pay any mind
-        // to region obligations that arise (there shouldn't really be any
-        // anyhow).
-        cause: ObligationCause::misc(span, CRATE_DEF_ID),
-        recursion_depth: 0,
-        predicate: pred.to_predicate(infcx.tcx),
-    };
+    let obligation = Obligation::new(infcx.tcx, ObligationCause::dummy(), param_env, pred);
 
     let result = infcx.evaluate_obligation_no_overflow(&obligation);
     debug!(?result);
@@ -166,14 +155,13 @@ fn pred_known_to_hold_modulo_regions<'tcx>(
         // this function's result remains infallible, we must confirm
         // that guess. While imperfect, I believe this is sound.
 
-        // FIXME(@lcnr): this function doesn't seem right.
-        //
         // The handling of regions in this area of the code is terrible,
         // see issue #29149. We should be able to improve on this with
         // NLL.
-        let errors = fully_solve_obligation(infcx, obligation);
-
-        match &errors[..] {
+        let ocx = ObligationCtxt::new(infcx);
+        ocx.register_obligation(obligation);
+        let errors = ocx.select_all_or_error();
+        match errors.as_slice() {
             [] => true,
             errors => {
                 debug!(?errors);
@@ -210,7 +198,7 @@ fn do_normalize_predicates<'tcx>(
     let predicates = match fully_normalize(&infcx, cause, elaborated_env, predicates) {
         Ok(predicates) => predicates,
         Err(errors) => {
-            let reported = infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+            let reported = infcx.err_ctxt().report_fulfillment_errors(&errors);
             return Err(reported);
         }
     };
@@ -276,9 +264,7 @@ pub fn normalize_param_env_or_error<'tcx>(
     // and errors will get reported then; so outside of type inference we
     // can be sure that no errors should occur.
     let mut predicates: Vec<_> =
-        util::elaborate_predicates(tcx, unnormalized_env.caller_bounds().into_iter())
-            .map(|obligation| obligation.predicate)
-            .collect();
+        util::elaborate(tcx, unnormalized_env.caller_bounds().into_iter()).collect();
 
     debug!("normalize_param_env_or_error: elaborated-predicates={:?}", predicates);
 
@@ -389,43 +375,6 @@ where
     Ok(resolved_value)
 }
 
-/// Process an obligation (and any nested obligations that come from it) to
-/// completion, returning any errors
-pub fn fully_solve_obligation<'tcx>(
-    infcx: &InferCtxt<'tcx>,
-    obligation: PredicateObligation<'tcx>,
-) -> Vec<FulfillmentError<'tcx>> {
-    fully_solve_obligations(infcx, [obligation])
-}
-
-/// Process a set of obligations (and any nested obligations that come from them)
-/// to completion
-pub fn fully_solve_obligations<'tcx>(
-    infcx: &InferCtxt<'tcx>,
-    obligations: impl IntoIterator<Item = PredicateObligation<'tcx>>,
-) -> Vec<FulfillmentError<'tcx>> {
-    let ocx = ObligationCtxt::new(infcx);
-    ocx.register_obligations(obligations);
-    ocx.select_all_or_error()
-}
-
-/// Process a bound (and any nested obligations that come from it) to completion.
-/// This is a convenience function for traits that have no generic arguments, such
-/// as auto traits, and builtin traits like Copy or Sized.
-pub fn fully_solve_bound<'tcx>(
-    infcx: &InferCtxt<'tcx>,
-    cause: ObligationCause<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
-    ty: Ty<'tcx>,
-    bound: DefId,
-) -> Vec<FulfillmentError<'tcx>> {
-    let tcx = infcx.tcx;
-    let trait_ref = tcx.mk_trait_ref(bound, [ty]);
-    let obligation = Obligation::new(tcx, cause, param_env, ty::Binder::dummy(trait_ref));
-
-    fully_solve_obligation(infcx, obligation)
-}
-
 /// Normalizes the predicates and checks whether they hold in an empty environment. If this
 /// returns true, then either normalize encountered an error or one of the predicates did not
 /// hold. Used when creating vtables to check for unsatisfiable methods.
diff --git a/compiler/rustc_trait_selection/src/traits/object_safety.rs b/compiler/rustc_trait_selection/src/traits/object_safety.rs
index 4eacb5211..b8ad1925e 100644
--- a/compiler/rustc_trait_selection/src/traits/object_safety.rs
+++ b/compiler/rustc_trait_selection/src/traits/object_safety.rs
@@ -8,20 +8,19 @@
 //!   - not reference the erased type `Self` except for in this receiver;
 //!   - not have generic type parameters.
 
-use super::{elaborate_predicates, elaborate_trait_ref};
+use super::elaborate;
 
 use crate::infer::TyCtxtInferExt;
 use crate::traits::query::evaluate_obligation::InferCtxtExt;
 use crate::traits::{self, Obligation, ObligationCause};
-use hir::def::DefKind;
 use rustc_errors::{DelayDm, FatalError, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::subst::{GenericArg, InternalSubsts};
-use rustc_middle::ty::ToPredicate;
 use rustc_middle::ty::{
     self, EarlyBinder, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor,
 };
+use rustc_middle::ty::{ToPredicate, TypeVisitableExt};
 use rustc_session::lint::builtin::WHERE_CLAUSES_OBJECT_SAFETY;
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
@@ -140,6 +139,10 @@ fn object_safety_violations_for_trait(
     if !spans.is_empty() {
         violations.push(ObjectSafetyViolation::SupertraitSelf(spans));
     }
+    let spans = super_predicates_have_non_lifetime_binders(tcx, trait_def_id);
+    if !spans.is_empty() {
+        violations.push(ObjectSafetyViolation::SupertraitNonLifetimeBinder(spans));
+    }
 
     violations.extend(
         tcx.associated_items(trait_def_id)
@@ -157,6 +160,7 @@ fn object_safety_violations_for_trait(
                 .in_definition_order()
                 .filter(|item| item.kind == ty::AssocKind::Type)
                 .filter(|item| !tcx.generics_of(item.def_id).params.is_empty())
+                .filter(|item| tcx.opt_rpitit_info(item.def_id).is_none())
                 .map(|item| {
                     let ident = item.ident(tcx);
                     ObjectSafetyViolation::GAT(ident.name, ident.span)
@@ -331,7 +335,7 @@ fn predicate_references_self<'tcx>(
             has_self_ty(&ty.into()).then_some(sp)
         }
 
-        ty::PredicateKind::AliasEq(..) => bug!("`AliasEq` not allowed as assumption"),
+        ty::PredicateKind::AliasRelate(..) => bug!("`AliasRelate` not allowed as assumption"),
 
         ty::PredicateKind::WellFormed(..)
         | ty::PredicateKind::ObjectSafe(..)
@@ -348,6 +352,21 @@ fn predicate_references_self<'tcx>(
     }
 }
 
+fn super_predicates_have_non_lifetime_binders(
+    tcx: TyCtxt<'_>,
+    trait_def_id: DefId,
+) -> SmallVec<[Span; 1]> {
+    // If non_lifetime_binders is disabled, then exit early
+    if !tcx.features().non_lifetime_binders {
+        return SmallVec::new();
+    }
+    tcx.super_predicates_of(trait_def_id)
+        .predicates
+        .iter()
+        .filter_map(|(pred, span)| pred.has_non_region_late_bound().then_some(*span))
+        .collect()
+}
+
 fn trait_has_sized_self(tcx: TyCtxt<'_>, trait_def_id: DefId) -> bool {
     generics_require_sized_self(tcx, trait_def_id)
 }
@@ -360,26 +379,24 @@ fn generics_require_sized_self(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
     // Search for a predicate like `Self : Sized` amongst the trait bounds.
     let predicates = tcx.predicates_of(def_id);
     let predicates = predicates.instantiate_identity(tcx).predicates;
-    elaborate_predicates(tcx, predicates.into_iter()).any(|obligation| {
-        match obligation.predicate.kind().skip_binder() {
-            ty::PredicateKind::Clause(ty::Clause::Trait(ref trait_pred)) => {
-                trait_pred.def_id() == sized_def_id && trait_pred.self_ty().is_param(0)
-            }
-            ty::PredicateKind::Clause(ty::Clause::Projection(..))
-            | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
-            | ty::PredicateKind::Subtype(..)
-            | ty::PredicateKind::Coerce(..)
-            | ty::PredicateKind::Clause(ty::Clause::RegionOutlives(..))
-            | ty::PredicateKind::WellFormed(..)
-            | ty::PredicateKind::ObjectSafe(..)
-            | ty::PredicateKind::ClosureKind(..)
-            | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(..))
-            | ty::PredicateKind::ConstEvaluatable(..)
-            | ty::PredicateKind::ConstEquate(..)
-            | ty::PredicateKind::AliasEq(..)
-            | ty::PredicateKind::Ambiguous
-            | ty::PredicateKind::TypeWellFormedFromEnv(..) => false,
+    elaborate(tcx, predicates.into_iter()).any(|pred| match pred.kind().skip_binder() {
+        ty::PredicateKind::Clause(ty::Clause::Trait(ref trait_pred)) => {
+            trait_pred.def_id() == sized_def_id && trait_pred.self_ty().is_param(0)
         }
+        ty::PredicateKind::Clause(ty::Clause::Projection(..))
+        | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
+        | ty::PredicateKind::Subtype(..)
+        | ty::PredicateKind::Coerce(..)
+        | ty::PredicateKind::Clause(ty::Clause::RegionOutlives(..))
+        | ty::PredicateKind::WellFormed(..)
+        | ty::PredicateKind::ObjectSafe(..)
+        | ty::PredicateKind::ClosureKind(..)
+        | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(..))
+        | ty::PredicateKind::ConstEvaluatable(..)
+        | ty::PredicateKind::ConstEquate(..)
+        | ty::PredicateKind::AliasRelate(..)
+        | ty::PredicateKind::Ambiguous
+        | ty::PredicateKind::TypeWellFormedFromEnv(..) => false,
     })
 }
 
@@ -649,10 +666,11 @@ fn object_ty_for_trait<'tcx>(
     });
     debug!(?trait_predicate);
 
-    let mut elaborated_predicates: Vec<_> = elaborate_trait_ref(tcx, trait_ref)
-        .filter_map(|obligation| {
-            debug!(?obligation);
-            let pred = obligation.predicate.to_opt_poly_projection_pred()?;
+    let pred: ty::Predicate<'tcx> = trait_ref.to_predicate(tcx);
+    let mut elaborated_predicates: Vec<_> = elaborate(tcx, [pred])
+        .filter_map(|pred| {
+            debug!(?pred);
+            let pred = pred.to_opt_poly_projection_pred()?;
             Some(pred.map_bound(|p| {
                 ty::ExistentialPredicate::Projection(ty::ExistentialProjection::erase_self_ty(
                     tcx, p,
@@ -854,7 +872,7 @@ fn contains_illegal_self_type_reference<'tcx, T: TypeVisitable<TyCtxt<'tcx>>>(
                     }
                 }
                 ty::Alias(ty::Projection, ref data)
-                    if self.tcx.def_kind(data.def_id) == DefKind::ImplTraitPlaceholder =>
+                    if self.tcx.is_impl_trait_in_trait(data.def_id) =>
                 {
                     // We'll deny these later in their own pass
                     ControlFlow::Continue(())
@@ -921,7 +939,7 @@ pub fn contains_illegal_impl_trait_in_trait<'tcx>(
     ty.skip_binder().walk().find_map(|arg| {
         if let ty::GenericArgKind::Type(ty) = arg.unpack()
             && let ty::Alias(ty::Projection, proj) = ty.kind()
-            && tcx.def_kind(proj.def_id) == DefKind::ImplTraitPlaceholder
+            && tcx.is_impl_trait_in_trait(proj.def_id)
         {
             Some(MethodViolationCode::ReferencesImplTraitInTrait(tcx.def_span(proj.def_id)))
         } else {
diff --git a/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs b/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
index 6cb64ad57..cff3d277a 100644
--- a/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
+++ b/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
@@ -1,9 +1,10 @@
 use crate::infer::InferCtxt;
 use crate::traits::query::type_op::{self, TypeOp, TypeOpOutput};
 use crate::traits::query::NoSolution;
-use crate::traits::ObligationCause;
+use crate::traits::{ObligationCause, ObligationCtxt};
 use rustc_data_structures::fx::FxIndexSet;
-use rustc_middle::ty::{self, ParamEnv, Ty};
+use rustc_infer::infer::resolve::OpportunisticRegionResolver;
+use rustc_middle::ty::{self, ParamEnv, Ty, TypeFolder, TypeVisitableExt};
 use rustc_span::def_id::LocalDefId;
 
 pub use rustc_middle::traits::query::OutlivesBound;
@@ -52,6 +53,10 @@ impl<'a, 'tcx: 'a> InferCtxtExt<'a, 'tcx> for InferCtxt<'tcx> {
         body_id: LocalDefId,
         ty: Ty<'tcx>,
     ) -> Vec<OutlivesBound<'tcx>> {
+        let ty = self.resolve_vars_if_possible(ty);
+        let ty = OpportunisticRegionResolver::new(self).fold_ty(ty);
+        assert!(!ty.needs_infer());
+
         let span = self.tcx.def_span(body_id);
         let result = param_env
             .and(type_op::implied_outlives_bounds::ImpliedOutlivesBounds { ty })
@@ -71,22 +76,23 @@ impl<'a, 'tcx: 'a> InferCtxtExt<'a, 'tcx> for InferCtxt<'tcx> {
 
         if let Some(constraints) = constraints {
             debug!(?constraints);
+            if !constraints.member_constraints.is_empty() {
+                span_bug!(span, "{:#?}", constraints.member_constraints);
+            }
+
             // Instantiation may have produced new inference variables and constraints on those
             // variables. Process these constraints.
+            let ocx = ObligationCtxt::new(self);
             let cause = ObligationCause::misc(span, body_id);
-            let errors = super::fully_solve_obligations(
-                self,
-                constraints.outlives.iter().map(|constraint| {
-                    self.query_outlives_constraint_to_obligation(
-                        *constraint,
-                        cause.clone(),
-                        param_env,
-                    )
-                }),
-            );
-            if !constraints.member_constraints.is_empty() {
-                span_bug!(span, "{:#?}", constraints.member_constraints);
+            for &constraint in &constraints.outlives {
+                ocx.register_obligation(self.query_outlives_constraint_to_obligation(
+                    constraint,
+                    cause.clone(),
+                    param_env,
+                ));
             }
+
+            let errors = ocx.select_all_or_error();
             if !errors.is_empty() {
                 self.tcx.sess.delay_span_bug(
                     span,
@@ -104,11 +110,6 @@ impl<'a, 'tcx: 'a> InferCtxtExt<'a, 'tcx> for InferCtxt<'tcx> {
         body_id: LocalDefId,
         tys: FxIndexSet<Ty<'tcx>>,
     ) -> Bounds<'a, 'tcx> {
-        tys.into_iter()
-            .map(move |ty| {
-                let ty = self.resolve_vars_if_possible(ty);
-                self.implied_outlives_bounds(param_env, body_id, ty)
-            })
-            .flatten()
+        tys.into_iter().flat_map(move |ty| self.implied_outlives_bounds(param_env, body_id, ty))
     }
 }
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index 870ecc2a9..826fc63ca 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -28,11 +28,11 @@ use rustc_hir::def::DefKind;
 use rustc_hir::lang_items::LangItem;
 use rustc_infer::infer::at::At;
 use rustc_infer::infer::resolve::OpportunisticRegionResolver;
+use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::traits::ImplSourceBuiltinData;
 use rustc_middle::traits::select::OverflowError;
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::visit::{MaxUniverse, TypeVisitable, TypeVisitableExt};
-use rustc_middle::ty::DefIdTree;
 use rustc_middle::ty::{self, Term, ToPredicate, Ty, TyCtxt};
 use rustc_span::symbol::sym;
 
@@ -286,12 +286,12 @@ fn project_and_unify_type<'cx, 'tcx>(
         );
     obligations.extend(new);
 
-    match infcx
-        .at(&obligation.cause, obligation.param_env)
-        // This is needed to support nested opaque types like `impl Fn() -> impl Trait`
-        .define_opaque_types(true)
-        .eq(normalized, actual)
-    {
+    // Need to define opaque types to support nested opaque types like `impl Fn() -> impl Trait`
+    match infcx.at(&obligation.cause, obligation.param_env).eq(
+        DefineOpaqueTypes::Yes,
+        normalized,
+        actual,
+    ) {
         Ok(InferOk { obligations: inferred_obligations, value: () }) => {
             obligations.extend(inferred_obligations);
             ProjectAndUnifyResult::Holds(obligations)
@@ -468,6 +468,11 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
             return ty;
         }
 
+        let (kind, data) = match *ty.kind() {
+            ty::Alias(kind, alias_ty) => (kind, alias_ty),
+            _ => return ty.super_fold_with(self),
+        };
+
         // We try to be a little clever here as a performance optimization in
         // cases where there are nested projections under binders.
         // For example:
@@ -491,13 +496,11 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
         // replace bound vars if the current type is a `Projection` and we need
         // to make sure we don't forget to fold the substs regardless.
 
-        match *ty.kind() {
+        match kind {
             // This is really important. While we *can* handle this, this has
             // severe performance implications for large opaque types with
             // late-bound regions. See `issue-88862` benchmark.
-            ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. })
-                if !substs.has_escaping_bound_vars() =>
-            {
+            ty::Opaque if !data.substs.has_escaping_bound_vars() => {
                 // Only normalize `impl Trait` outside of type inference, usually in codegen.
                 match self.param_env.reveal() {
                     Reveal::UserFacing => ty.super_fold_with(self),
@@ -513,8 +516,8 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
                             );
                         }
 
-                        let substs = substs.fold_with(self);
-                        let generic_ty = self.interner().type_of(def_id);
+                        let substs = data.substs.fold_with(self);
+                        let generic_ty = self.interner().type_of(data.def_id);
                         let concrete_ty = generic_ty.subst(self.interner(), substs);
                         self.depth += 1;
                         let folded_ty = self.fold_ty(concrete_ty);
@@ -523,8 +526,9 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
                     }
                 }
             }
+            ty::Opaque => ty.super_fold_with(self),
 
-            ty::Alias(ty::Projection, data) if !data.has_escaping_bound_vars() => {
+            ty::Projection if !data.has_escaping_bound_vars() => {
                 // This branch is *mostly* just an optimization: when we don't
                 // have escaping bound vars, we don't need to replace them with
                 // placeholders (see branch below). *Also*, we know that we can
@@ -563,7 +567,7 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
                 normalized_ty.ty().unwrap()
             }
 
-            ty::Alias(ty::Projection, data) => {
+            ty::Projection => {
                 // If there are escaping bound vars, we temporarily replace the
                 // bound vars with placeholders. Note though, that in the case
                 // that we still can't project for whatever reason (e.g. self
@@ -612,8 +616,6 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
                 );
                 normalized_ty
             }
-
-            _ => ty.super_fold_with(self),
         }
     }
 
@@ -770,7 +772,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for BoundVarReplacer<'_, 'tcx> {
             }
             ty::ReLateBound(debruijn, br) if debruijn >= self.current_index => {
                 let universe = self.universe_for(debruijn);
-                let p = ty::PlaceholderRegion { universe, name: br.kind };
+                let p = ty::PlaceholderRegion { universe, bound: br };
                 self.mapped_regions.insert(p, br);
                 self.infcx.tcx.mk_re_placeholder(p)
             }
@@ -788,7 +790,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for BoundVarReplacer<'_, 'tcx> {
             }
             ty::Bound(debruijn, bound_ty) if debruijn >= self.current_index => {
                 let universe = self.universe_for(debruijn);
-                let p = ty::PlaceholderType { universe, name: bound_ty.kind };
+                let p = ty::PlaceholderType { universe, bound: bound_ty };
                 self.mapped_types.insert(p, bound_ty);
                 self.infcx.tcx.mk_placeholder(p)
             }
@@ -807,7 +809,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for BoundVarReplacer<'_, 'tcx> {
             }
             ty::ConstKind::Bound(debruijn, bound_const) if debruijn >= self.current_index => {
                 let universe = self.universe_for(debruijn);
-                let p = ty::PlaceholderConst { universe, name: bound_const };
+                let p = ty::PlaceholderConst { universe, bound: bound_const };
                 self.mapped_consts.insert(p, bound_const);
                 self.infcx.tcx.mk_const(p, ct.ty())
             }
@@ -871,12 +873,12 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for PlaceholderReplacer<'_, 'tcx> {
 
     fn fold_region(&mut self, r0: ty::Region<'tcx>) -> ty::Region<'tcx> {
         let r1 = match *r0 {
-            ty::ReVar(_) => self
+            ty::ReVar(vid) => self
                 .infcx
                 .inner
                 .borrow_mut()
                 .unwrap_region_constraints()
-                .opportunistic_resolve_region(self.infcx.tcx, r0),
+                .opportunistic_resolve_var(self.infcx.tcx, vid),
             _ => r0,
         };
 
@@ -1296,7 +1298,7 @@ fn assemble_candidate_for_impl_trait_in_trait<'cx, 'tcx>(
 ) {
     let tcx = selcx.tcx();
     if tcx.def_kind(obligation.predicate.def_id) == DefKind::ImplTraitPlaceholder {
-        let trait_fn_def_id = tcx.impl_trait_in_trait_parent(obligation.predicate.def_id);
+        let trait_fn_def_id = tcx.impl_trait_in_trait_parent_fn(obligation.predicate.def_id);
 
         let trait_def_id = tcx.parent(trait_fn_def_id);
         let trait_substs =
@@ -2065,7 +2067,11 @@ fn confirm_param_env_candidate<'cx, 'tcx>(
 
     debug!(?cache_projection, ?obligation_projection);
 
-    match infcx.at(cause, param_env).eq(cache_projection, obligation_projection) {
+    match infcx.at(cause, param_env).eq(
+        DefineOpaqueTypes::No,
+        cache_projection,
+        obligation_projection,
+    ) {
         Ok(InferOk { value: _, obligations }) => {
             nested_obligations.extend(obligations);
             assoc_ty_own_obligations(selcx, obligation, &mut nested_obligations);
@@ -2194,13 +2200,14 @@ fn confirm_impl_trait_in_trait_candidate<'tcx>(
     let tcx = selcx.tcx();
     let mut obligations = data.nested;
 
-    let trait_fn_def_id = tcx.impl_trait_in_trait_parent(obligation.predicate.def_id);
+    let trait_fn_def_id = tcx.impl_trait_in_trait_parent_fn(obligation.predicate.def_id);
     let leaf_def = match specialization_graph::assoc_def(tcx, data.impl_def_id, trait_fn_def_id) {
         Ok(assoc_ty) => assoc_ty,
         Err(guar) => return Progress::error(tcx, guar),
     };
     // We don't support specialization for RPITITs anyways... yet.
-    if !leaf_def.is_final() {
+    // Also don't try to project to an RPITIT that has no value
+    if !leaf_def.is_final() || !leaf_def.item.defaultness(tcx).has_value() {
         return Progress { term: tcx.ty_error_misc().into(), obligations };
     }
 
diff --git a/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs b/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs
index f183248f2..edbe2de81 100644
--- a/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs
@@ -1,9 +1,8 @@
+use rustc_infer::traits::{TraitEngine, TraitEngineExt};
 use rustc_middle::ty;
-use rustc_session::config::TraitSolver;
 
 use crate::infer::canonical::OriginalQueryValues;
 use crate::infer::InferCtxt;
-use crate::solve::{Certainty, Goal, InferCtxtEvalExt, MaybeCause};
 use crate::traits::{EvaluationResult, OverflowError, PredicateObligation, SelectionContext};
 
 pub trait InferCtxtExt<'tcx> {
@@ -79,37 +78,30 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
             _ => obligation.param_env.without_const(),
         };
 
-        if self.tcx.sess.opts.unstable_opts.trait_solver != TraitSolver::Next {
+        if self.tcx.trait_solver_next() {
+            self.probe(|snapshot| {
+                let mut fulfill_cx = crate::solve::FulfillmentCtxt::new();
+                fulfill_cx.register_predicate_obligation(self, obligation.clone());
+                // True errors
+                // FIXME(-Ztrait-solver=next): Overflows are reported as ambig here, is that OK?
+                if !fulfill_cx.select_where_possible(self).is_empty() {
+                    Ok(EvaluationResult::EvaluatedToErr)
+                } else if !fulfill_cx.select_all_or_error(self).is_empty() {
+                    Ok(EvaluationResult::EvaluatedToAmbig)
+                } else if self.opaque_types_added_in_snapshot(snapshot) {
+                    Ok(EvaluationResult::EvaluatedToOkModuloOpaqueTypes)
+                } else if self.region_constraints_added_in_snapshot(snapshot).is_some() {
+                    Ok(EvaluationResult::EvaluatedToOkModuloRegions)
+                } else {
+                    Ok(EvaluationResult::EvaluatedToOk)
+                }
+            })
+        } else {
             let c_pred = self.canonicalize_query_keep_static(
                 param_env.and(obligation.predicate),
                 &mut _orig_values,
             );
             self.tcx.at(obligation.cause.span()).evaluate_obligation(c_pred)
-        } else {
-            self.probe(|snapshot| {
-                if let Ok((_, certainty)) =
-                    self.evaluate_root_goal(Goal::new(self.tcx, param_env, obligation.predicate))
-                {
-                    match certainty {
-                        Certainty::Yes => {
-                            if self.opaque_types_added_in_snapshot(snapshot) {
-                                Ok(EvaluationResult::EvaluatedToOkModuloOpaqueTypes)
-                            } else if self.region_constraints_added_in_snapshot(snapshot).is_some()
-                            {
-                                Ok(EvaluationResult::EvaluatedToOkModuloRegions)
-                            } else {
-                                Ok(EvaluationResult::EvaluatedToOk)
-                            }
-                        }
-                        Certainty::Maybe(MaybeCause::Ambiguity) => {
-                            Ok(EvaluationResult::EvaluatedToAmbig)
-                        }
-                        Certainty::Maybe(MaybeCause::Overflow) => Err(OverflowError::Canonical),
-                    }
-                } else {
-                    Ok(EvaluationResult::EvaluatedToErr)
-                }
-            })
         }
     }
 
diff --git a/compiler/rustc_trait_selection/src/traits/query/normalize.rs b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
index b0cec3ce7..a986a9b6a 100644
--- a/compiler/rustc_trait_selection/src/traits/query/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
@@ -197,23 +197,30 @@ impl<'cx, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for QueryNormalizer<'cx, 'tcx>
             return Ok(*ty);
         }
 
+        let (kind, data) = match *ty.kind() {
+            ty::Alias(kind, data) => (kind, data),
+            _ => {
+                let res = ty.try_super_fold_with(self)?;
+                self.cache.insert(ty, res);
+                return Ok(res);
+            }
+        };
+
         // See note in `rustc_trait_selection::traits::project` about why we
         // wait to fold the substs.
 
         // Wrap this in a closure so we don't accidentally return from the outer function
-        let res = match *ty.kind() {
+        let res = match kind {
             // This is really important. While we *can* handle this, this has
             // severe performance implications for large opaque types with
             // late-bound regions. See `issue-88862` benchmark.
-            ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. })
-                if !substs.has_escaping_bound_vars() =>
-            {
+            ty::Opaque if !data.substs.has_escaping_bound_vars() => {
                 // Only normalize `impl Trait` outside of type inference, usually in codegen.
                 match self.param_env.reveal() {
                     Reveal::UserFacing => ty.try_super_fold_with(self)?,
 
                     Reveal::All => {
-                        let substs = substs.try_fold_with(self)?;
+                        let substs = data.substs.try_fold_with(self)?;
                         let recursion_limit = self.interner().recursion_limit();
                         if !recursion_limit.value_within_limit(self.anon_depth) {
                             // A closure or generator may have itself as in its upvars.
@@ -228,7 +235,7 @@ impl<'cx, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for QueryNormalizer<'cx, 'tcx>
                             return ty.try_super_fold_with(self);
                         }
 
-                        let generic_ty = self.interner().type_of(def_id);
+                        let generic_ty = self.interner().type_of(data.def_id);
                         let concrete_ty = generic_ty.subst(self.interner(), substs);
                         self.anon_depth += 1;
                         if concrete_ty == ty {
@@ -248,62 +255,22 @@ impl<'cx, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for QueryNormalizer<'cx, 'tcx>
                 }
             }
 
-            ty::Alias(ty::Projection, data) if !data.has_escaping_bound_vars() => {
-                // This branch is just an optimization: when we don't have escaping bound vars,
-                // we don't need to replace them with placeholders (see branch below).
-
-                let tcx = self.infcx.tcx;
-                let data = data.try_fold_with(self)?;
-
-                let mut orig_values = OriginalQueryValues::default();
-                // HACK(matthewjasper) `'static` is special-cased in selection,
-                // so we cannot canonicalize it.
-                let c_data = self
-                    .infcx
-                    .canonicalize_query_keep_static(self.param_env.and(data), &mut orig_values);
-                debug!("QueryNormalizer: c_data = {:#?}", c_data);
-                debug!("QueryNormalizer: orig_values = {:#?}", orig_values);
-                let result = tcx.normalize_projection_ty(c_data)?;
-                // We don't expect ambiguity.
-                if result.is_ambiguous() {
-                    // Rustdoc normalizes possibly not well-formed types, so only
-                    // treat this as a bug if we're not in rustdoc.
-                    if !tcx.sess.opts.actually_rustdoc {
-                        tcx.sess.delay_span_bug(
-                            DUMMY_SP,
-                            format!("unexpected ambiguity: {:?} {:?}", c_data, result),
-                        );
-                    }
-                    return Err(NoSolution);
-                }
-                let InferOk { value: result, obligations } =
-                    self.infcx.instantiate_query_response_and_region_obligations(
-                        self.cause,
-                        self.param_env,
-                        &orig_values,
-                        result,
-                    )?;
-                debug!("QueryNormalizer: result = {:#?}", result);
-                debug!("QueryNormalizer: obligations = {:#?}", obligations);
-                self.obligations.extend(obligations);
-
-                let res = result.normalized_ty;
-                // `tcx.normalize_projection_ty` may normalize to a type that still has
-                // unevaluated consts, so keep normalizing here if that's the case.
-                if res != ty && res.has_type_flags(ty::TypeFlags::HAS_CT_PROJECTION) {
-                    res.try_super_fold_with(self)?
-                } else {
-                    res
-                }
-            }
+            ty::Opaque => ty.try_super_fold_with(self)?,
 
-            ty::Alias(ty::Projection, data) => {
+            ty::Projection => {
                 // See note in `rustc_trait_selection::traits::project`
 
                 let tcx = self.infcx.tcx;
                 let infcx = self.infcx;
-                let (data, mapped_regions, mapped_types, mapped_consts) =
-                    BoundVarReplacer::replace_bound_vars(infcx, &mut self.universes, data);
+                // Just an optimization: When we don't have escaping bound vars,
+                // we don't need to replace them with placeholders.
+                let (data, maps) = if data.has_escaping_bound_vars() {
+                    let (data, mapped_regions, mapped_types, mapped_consts) =
+                        BoundVarReplacer::replace_bound_vars(infcx, &mut self.universes, data);
+                    (data, Some((mapped_regions, mapped_types, mapped_consts)))
+                } else {
+                    (data, None)
+                };
                 let data = data.try_fold_with(self)?;
 
                 let mut orig_values = OriginalQueryValues::default();
@@ -337,14 +304,18 @@ impl<'cx, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for QueryNormalizer<'cx, 'tcx>
                 debug!("QueryNormalizer: result = {:#?}", result);
                 debug!("QueryNormalizer: obligations = {:#?}", obligations);
                 self.obligations.extend(obligations);
-                let res = PlaceholderReplacer::replace_placeholders(
-                    infcx,
-                    mapped_regions,
-                    mapped_types,
-                    mapped_consts,
-                    &self.universes,
-                    result.normalized_ty,
-                );
+                let res = if let Some((mapped_regions, mapped_types, mapped_consts)) = maps {
+                    PlaceholderReplacer::replace_placeholders(
+                        infcx,
+                        mapped_regions,
+                        mapped_types,
+                        mapped_consts,
+                        &self.universes,
+                        result.normalized_ty,
+                    )
+                } else {
+                    result.normalized_ty
+                };
                 // `tcx.normalize_projection_ty` may normalize to a type that still has
                 // unevaluated consts, so keep normalizing here if that's the case.
                 if res != ty && res.has_type_flags(ty::TypeFlags::HAS_CT_PROJECTION) {
@@ -353,8 +324,6 @@ impl<'cx, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for QueryNormalizer<'cx, 'tcx>
                     res
                 }
             }
-
-            _ => ty.try_super_fold_with(self)?,
         };
 
         self.cache.insert(ty, res);
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs
index 6bf3ed0d0..8f1b05c11 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs
@@ -1,8 +1,8 @@
 use crate::infer::canonical::query_response;
 use crate::infer::{InferCtxt, InferOk};
-use crate::traits;
 use crate::traits::query::type_op::TypeOpOutput;
 use crate::traits::query::Fallible;
+use crate::traits::ObligationCtxt;
 use rustc_infer::infer::region_constraints::RegionConstraintData;
 use rustc_span::source_map::DUMMY_SP;
 
@@ -73,7 +73,9 @@ pub fn scrape_region_constraints<'tcx, Op: super::TypeOp<'tcx, Output = R>, R>(
     );
 
     let InferOk { value, obligations } = infcx.commit_if_ok(|_| op())?;
-    let errors = traits::fully_solve_obligations(infcx, obligations);
+    let ocx = ObligationCtxt::new(infcx);
+    ocx.register_obligations(obligations);
+    let errors = ocx.select_all_or_error();
     if !errors.is_empty() {
         infcx.tcx.sess.diagnostic().delay_span_bug(
             DUMMY_SP,
@@ -82,9 +84,7 @@ pub fn scrape_region_constraints<'tcx, Op: super::TypeOp<'tcx, Output = R>, R>(
     }
 
     let region_obligations = infcx.take_registered_region_obligations();
-
     let region_constraint_data = infcx.take_and_reset_region_constraints();
-
     let region_constraints = query_response::make_query_region_constraints(
         infcx.tcx,
         region_obligations
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index e91057356..1f5bbc178 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -5,12 +5,14 @@
 //! candidates. See the [rustc dev guide] for more details.
 //!
 //! [rustc dev guide]:https://rustc-dev-guide.rust-lang.org/traits/resolution.html#candidate-assembly
+
+use hir::def_id::DefId;
 use hir::LangItem;
 use rustc_hir as hir;
 use rustc_infer::traits::ObligationCause;
 use rustc_infer::traits::{Obligation, SelectionError, TraitObligation};
+use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams, TreatProjections};
 use rustc_middle::ty::{self, Ty, TypeVisitableExt};
-use rustc_target::spec::abi::Abi;
 
 use crate::traits;
 use crate::traits::query::evaluate_obligation::InferCtxtExt;
@@ -95,7 +97,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             } else if lang_items.tuple_trait() == Some(def_id) {
                 self.assemble_candidate_for_tuple(obligation, &mut candidates);
             } else if lang_items.pointer_like() == Some(def_id) {
-                self.assemble_candidate_for_ptr_sized(obligation, &mut candidates);
+                self.assemble_candidate_for_pointer_like(obligation, &mut candidates);
+            } else if lang_items.fn_ptr_trait() == Some(def_id) {
+                self.assemble_candidates_for_fn_ptr_trait(obligation, &mut candidates);
             } else {
                 if lang_items.clone_trait() == Some(def_id) {
                     // Same builtin conditions as `Copy`, i.e., every type which has builtin support
@@ -290,6 +294,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             return;
         }
 
+        // Keep this function in sync with extract_tupled_inputs_and_output_from_callable
+        // until the old solver (and thus this function) is removed.
+
         // Okay to skip binder because what we are inspecting doesn't involve bound regions.
         let self_ty = obligation.self_ty().skip_binder();
         match *self_ty.kind() {
@@ -298,31 +305,19 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 candidates.ambiguous = true; // Could wind up being a fn() type.
             }
             // Provide an impl, but only for suitable `fn` pointers.
-            ty::FnPtr(_) => {
-                if let ty::FnSig {
-                    unsafety: hir::Unsafety::Normal,
-                    abi: Abi::Rust,
-                    c_variadic: false,
-                    ..
-                } = self_ty.fn_sig(self.tcx()).skip_binder()
-                {
+            ty::FnPtr(sig) => {
+                if sig.is_fn_trait_compatible() {
                     candidates.vec.push(FnPointerCandidate { is_const: false });
                 }
             }
             // Provide an impl for suitable functions, rejecting `#[target_feature]` functions (RFC 2396).
             ty::FnDef(def_id, _) => {
-                if let ty::FnSig {
-                    unsafety: hir::Unsafety::Normal,
-                    abi: Abi::Rust,
-                    c_variadic: false,
-                    ..
-                } = self_ty.fn_sig(self.tcx()).skip_binder()
+                if self.tcx().fn_sig(def_id).skip_binder().is_fn_trait_compatible()
+                    && self.tcx().codegen_fn_attrs(def_id).target_features.is_empty()
                 {
-                    if self.tcx().codegen_fn_attrs(def_id).target_features.is_empty() {
-                        candidates
-                            .vec
-                            .push(FnPointerCandidate { is_const: self.tcx().is_const_fn(def_id) });
-                    }
+                    candidates
+                        .vec
+                        .push(FnPointerCandidate { is_const: self.tcx().is_const_fn(def_id) });
                 }
             }
             _ => {}
@@ -330,13 +325,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     }
 
     /// Searches for impls that might apply to `obligation`.
+    #[instrument(level = "debug", skip(self, candidates))]
     fn assemble_candidates_from_impls(
         &mut self,
         obligation: &TraitObligation<'tcx>,
         candidates: &mut SelectionCandidateSet<'tcx>,
     ) {
-        debug!(?obligation, "assemble_candidates_from_impls");
-
         // Essentially any user-written impl will match with an error type,
         // so creating `ImplCandidates` isn't useful. However, we might
         // end up finding a candidate elsewhere (e.g. a `BuiltinCandidate` for `Sized`)
@@ -350,6 +344,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             return;
         }
 
+        let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::ForLookup };
+        let obligation_substs = obligation.predicate.skip_binder().trait_ref.substs;
         self.tcx().for_each_relevant_impl(
             obligation.predicate.def_id(),
             obligation.predicate.skip_binder().trait_ref.self_ty(),
@@ -358,7 +354,14 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 // consider a "quick reject". This avoids creating more types
                 // and so forth that we need to.
                 let impl_trait_ref = self.tcx().impl_trait_ref(impl_def_id).unwrap();
-                if self.fast_reject_trait_refs(obligation, &impl_trait_ref.0) {
+                if !drcx.substs_refs_may_unify(obligation_substs, impl_trait_ref.0.substs) {
+                    return;
+                }
+                if self.reject_fn_ptr_impls(
+                    impl_def_id,
+                    obligation,
+                    impl_trait_ref.skip_binder().self_ty(),
+                ) {
                     return;
                 }
 
@@ -371,6 +374,99 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         );
     }
 
+    /// The various `impl<T: FnPtr> Trait for T` in libcore are more like builtin impls for all function items
+    /// and function pointers and less like blanket impls. Rejecting them when they can't possibly apply (because
+    /// the obligation's self-type does not implement `FnPtr`) avoids reporting that the self type does not implement
+    /// `FnPtr`, when we wanted to report that it doesn't implement `Trait`.
+    #[instrument(level = "trace", skip(self), ret)]
+    fn reject_fn_ptr_impls(
+        &self,
+        impl_def_id: DefId,
+        obligation: &TraitObligation<'tcx>,
+        impl_self_ty: Ty<'tcx>,
+    ) -> bool {
+        // Let `impl<T: FnPtr> Trait for Vec<T>` go through the normal rejection path.
+        if !matches!(impl_self_ty.kind(), ty::Param(..)) {
+            return false;
+        }
+        let Some(fn_ptr_trait) = self.tcx().lang_items().fn_ptr_trait() else {
+            return false;
+        };
+
+        for &(predicate, _) in self.tcx().predicates_of(impl_def_id).predicates {
+            let ty::PredicateKind::Clause(ty::Clause::Trait(pred))
+                = predicate.kind().skip_binder() else { continue };
+            if fn_ptr_trait != pred.trait_ref.def_id {
+                continue;
+            }
+            trace!(?pred);
+            // Not the bound we're looking for
+            if pred.self_ty() != impl_self_ty {
+                continue;
+            }
+
+            match obligation.self_ty().skip_binder().kind() {
+                // Fast path to avoid evaluating an obligation that trivially holds.
+                // There may be more bounds, but these are checked by the regular path.
+                ty::FnPtr(..) => return false,
+                // These may potentially implement `FnPtr`
+                ty::Placeholder(..)
+                | ty::Dynamic(_, _, _)
+                | ty::Alias(_, _)
+                | ty::Infer(_)
+                | ty::Param(..) => {}
+
+                ty::Bound(_, _) => span_bug!(
+                    obligation.cause.span(),
+                    "cannot have escaping bound var in self type of {obligation:#?}"
+                ),
+                // These can't possibly implement `FnPtr` as they are concrete types
+                // and not `FnPtr`
+                ty::Bool
+                | ty::Char
+                | ty::Int(_)
+                | ty::Uint(_)
+                | ty::Float(_)
+                | ty::Adt(_, _)
+                | ty::Foreign(_)
+                | ty::Str
+                | ty::Array(_, _)
+                | ty::Slice(_)
+                | ty::RawPtr(_)
+                | ty::Ref(_, _, _)
+                | ty::Closure(_, _)
+                | ty::Generator(_, _, _)
+                | ty::GeneratorWitness(_)
+                | ty::GeneratorWitnessMIR(_, _)
+                | ty::Never
+                | ty::Tuple(_)
+                | ty::Error(_) => return true,
+                // FIXME: Function definitions could actually implement `FnPtr` by
+                // casting the ZST function def to a function pointer.
+                ty::FnDef(_, _) => return true,
+            }
+
+            // Generic params can implement `FnPtr` if the predicate
+            // holds within its own environment.
+            let obligation = Obligation::new(
+                self.tcx(),
+                obligation.cause.clone(),
+                obligation.param_env,
+                self.tcx().mk_predicate(obligation.predicate.map_bound(|mut pred| {
+                    pred.trait_ref =
+                        self.tcx().mk_trait_ref(fn_ptr_trait, [pred.trait_ref.self_ty()]);
+                    ty::PredicateKind::Clause(ty::Clause::Trait(pred))
+                })),
+            );
+            if let Ok(r) = self.infcx.evaluate_obligation(&obligation) {
+                if !r.may_apply() {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
     fn assemble_candidates_from_auto_impls(
         &mut self,
         obligation: &TraitObligation<'tcx>,
@@ -783,6 +879,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 let relevant_impl = self.tcx().find_map_relevant_impl(
                     self.tcx().require_lang_item(LangItem::Drop, None),
                     obligation.predicate.skip_binder().trait_ref.self_ty(),
+                    TreatProjections::ForLookup,
                     Some,
                 );
 
@@ -845,15 +942,15 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         }
     }
 
-    fn assemble_candidate_for_ptr_sized(
+    fn assemble_candidate_for_pointer_like(
         &mut self,
         obligation: &TraitObligation<'tcx>,
         candidates: &mut SelectionCandidateSet<'tcx>,
     ) {
         // The regions of a type don't affect the size of the type
-        let self_ty = self
-            .tcx()
-            .erase_regions(self.tcx().erase_late_bound_regions(obligation.predicate.self_ty()));
+        let tcx = self.tcx();
+        let self_ty =
+            tcx.erase_regions(tcx.erase_late_bound_regions(obligation.predicate.self_ty()));
 
         // But if there are inference variables, we have to wait until it's resolved.
         if self_ty.has_non_region_infer() {
@@ -861,13 +958,55 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             return;
         }
 
-        let usize_layout =
-            self.tcx().layout_of(ty::ParamEnv::empty().and(self.tcx().types.usize)).unwrap().layout;
-        if let Ok(layout) = self.tcx().layout_of(obligation.param_env.and(self_ty))
-            && layout.layout.size() == usize_layout.size()
-            && layout.layout.align().abi == usize_layout.align().abi
+        if let Ok(layout) = tcx.layout_of(obligation.param_env.and(self_ty))
+            && layout.layout.is_pointer_like(&tcx.data_layout)
         {
             candidates.vec.push(BuiltinCandidate { has_nested: false });
         }
     }
+
+    fn assemble_candidates_for_fn_ptr_trait(
+        &mut self,
+        obligation: &TraitObligation<'tcx>,
+        candidates: &mut SelectionCandidateSet<'tcx>,
+    ) {
+        let self_ty = self.infcx.shallow_resolve(obligation.self_ty());
+        match self_ty.skip_binder().kind() {
+            ty::FnPtr(_) => candidates.vec.push(BuiltinCandidate { has_nested: false }),
+            ty::Bool
+            | ty::Char
+            | ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Adt(..)
+            | ty::Foreign(..)
+            | ty::Str
+            | ty::Array(..)
+            | ty::Slice(_)
+            | ty::RawPtr(_)
+            | ty::Ref(..)
+            | ty::FnDef(..)
+            | ty::Placeholder(..)
+            | ty::Dynamic(..)
+            | ty::Closure(..)
+            | ty::Generator(..)
+            | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
+            | ty::Never
+            | ty::Tuple(..)
+            | ty::Alias(..)
+            | ty::Param(..)
+            | ty::Bound(..)
+            | ty::Error(_)
+            | ty::Infer(
+                ty::InferTy::IntVar(_)
+                | ty::InferTy::FloatVar(_)
+                | ty::InferTy::FreshIntTy(_)
+                | ty::InferTy::FreshFloatTy(_),
+            ) => {}
+            ty::Infer(ty::InferTy::TyVar(_) | ty::InferTy::FreshTy(_)) => {
+                candidates.ambiguous = true;
+            }
+        }
+    }
 }
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index 21c158fd0..88121f865 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -8,8 +8,8 @@
 //! https://rustc-dev-guide.rust-lang.org/traits/resolution.html#confirmation
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir::lang_items::LangItem;
-use rustc_infer::infer::InferOk;
 use rustc_infer::infer::LateBoundRegionConversionTime::HigherRankedType;
+use rustc_infer::infer::{DefineOpaqueTypes, InferOk};
 use rustc_middle::ty::{
     self, Binder, GenericParamDefKind, InternalSubsts, SubstsRef, ToPolyTraitRef, ToPredicate,
     TraitRef, Ty, TyCtxt, TypeVisitableExt,
@@ -18,7 +18,7 @@ use rustc_session::config::TraitSolver;
 use rustc_span::def_id::DefId;
 
 use crate::traits::project::{normalize_with_depth, normalize_with_depth_to};
-use crate::traits::util::{self, closure_trait_ref_and_return_type, predicate_for_trait_def};
+use crate::traits::util::{self, closure_trait_ref_and_return_type};
 use crate::traits::vtable::{
     count_own_vtable_entries, prepare_vtable_segments, vtable_trait_first_method_offset,
     VtblSegment,
@@ -131,6 +131,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             }
         };
 
+        // The obligations returned by confirmation are recursively evaluated
+        // so we need to make sure they have the correct depth.
+        for subobligation in impl_src.borrow_nested_obligations_mut() {
+            subobligation.set_depth_from_parent(obligation.recursion_depth);
+        }
+
         if !obligation.predicate.is_const_if_const() {
             // normalize nested predicates according to parent predicate's constness.
             impl_src = impl_src.map(|mut o| {
@@ -177,7 +183,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         obligations.extend(self.infcx.commit_if_ok(|_| {
             self.infcx
                 .at(&obligation.cause, obligation.param_env)
-                .sup(placeholder_trait_predicate, candidate)
+                .sup(DefineOpaqueTypes::No, placeholder_trait_predicate, candidate)
                 .map(|InferOk { obligations, .. }| obligations)
                 .map_err(|_| Unimplemented)
         })?);
@@ -253,15 +259,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             };
 
             let cause = obligation.derived_cause(BuiltinDerivedObligation);
-            ensure_sufficient_stack(|| {
-                self.collect_predicates_for_types(
-                    obligation.param_env,
-                    cause,
-                    obligation.recursion_depth + 1,
-                    trait_def,
-                    nested,
-                )
-            })
+            self.collect_predicates_for_types(
+                obligation.param_env,
+                cause,
+                obligation.recursion_depth + 1,
+                trait_def,
+                nested,
+            )
         } else {
             vec![]
         };
@@ -462,7 +466,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         nested.extend(self.infcx.commit_if_ok(|_| {
             self.infcx
                 .at(&obligation.cause, obligation.param_env)
-                .sup(obligation_trait_ref, upcast_trait_ref)
+                .sup(DefineOpaqueTypes::No, obligation_trait_ref, upcast_trait_ref)
                 .map(|InferOk { obligations, .. }| obligations)
                 .map_err(|_| Unimplemented)
         })?);
@@ -601,10 +605,18 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         debug!(?obligation, "confirm_fn_pointer_candidate");
 
         let tcx = self.tcx();
-        let self_ty = self
+
+        let Some(self_ty) = self
             .infcx
-            .shallow_resolve(obligation.self_ty().no_bound_vars())
-            .expect("fn pointer should not capture bound vars from predicate");
+            .shallow_resolve(obligation.self_ty().no_bound_vars()) else
+        {
+            // FIXME: Ideally we'd support `for<'a> fn(&'a ()): Fn(&'a ())`,
+            // but we do not currently. Luckily, such a bound is not
+            // particularly useful, so we don't expect users to write
+            // them often.
+            return Err(SelectionError::Unimplemented);
+        };
+
         let sig = self_ty.fn_sig(tcx);
         let trait_ref = closure_trait_ref_and_return_type(
             tcx,
@@ -819,11 +831,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 )
             });
 
+        // needed to define opaque types for tests/ui/type-alias-impl-trait/assoc-projection-ice.rs
         self.infcx
             .at(&obligation.cause, obligation.param_env)
-            // needed for tests/ui/type-alias-impl-trait/assoc-projection-ice.rs
-            .define_opaque_types(true)
-            .sup(obligation_trait_ref, expected_trait_ref)
+            .sup(DefineOpaqueTypes::Yes, obligation_trait_ref, expected_trait_ref)
             .map(|InferOk { mut obligations, .. }| {
                 obligations.extend(nested);
                 obligations
@@ -888,7 +899,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 let InferOk { obligations, .. } = self
                     .infcx
                     .at(&obligation.cause, obligation.param_env)
-                    .sup(target, source_trait)
+                    .sup(DefineOpaqueTypes::No, target, source_trait)
                     .map_err(|_| Unimplemented)?;
                 nested.extend(obligations);
 
@@ -987,7 +998,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 let InferOk { obligations, .. } = self
                     .infcx
                     .at(&obligation.cause, obligation.param_env)
-                    .sup(target, source_trait)
+                    .sup(DefineOpaqueTypes::No, target, source_trait)
                     .map_err(|_| Unimplemented)?;
                 nested.extend(obligations);
 
@@ -1058,7 +1069,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 let InferOk { obligations, .. } = self
                     .infcx
                     .at(&obligation.cause, obligation.param_env)
-                    .eq(b, a)
+                    .eq(DefineOpaqueTypes::No, b, a)
                     .map_err(|_| Unimplemented)?;
                 nested.extend(obligations);
             }
@@ -1073,6 +1084,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 let tail_field = def
                     .non_enum_variant()
                     .fields
+                    .raw
                     .last()
                     .expect("expected unsized ADT to have a tail field");
                 let tail_field_ty = tcx.type_of(tail_field.did);
@@ -1106,19 +1118,16 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 let InferOk { obligations, .. } = self
                     .infcx
                     .at(&obligation.cause, obligation.param_env)
-                    .eq(target, new_struct)
+                    .eq(DefineOpaqueTypes::No, target, new_struct)
                     .map_err(|_| Unimplemented)?;
                 nested.extend(obligations);
 
                 // Construct the nested `TailField<T>: Unsize<TailField<U>>` predicate.
-                nested.push(predicate_for_trait_def(
+                let tail_unsize_obligation = obligation.with(
                     tcx,
-                    obligation.param_env,
-                    obligation.cause.clone(),
-                    obligation.predicate.def_id(),
-                    obligation.recursion_depth + 1,
-                    [source_tail, target_tail],
-                ));
+                    tcx.mk_trait_ref(obligation.predicate.def_id(), [source_tail, target_tail]),
+                );
+                nested.push(tail_unsize_obligation);
             }
 
             // `(.., T)` -> `(.., U)`
@@ -1136,21 +1145,14 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 let InferOk { obligations, .. } = self
                     .infcx
                     .at(&obligation.cause, obligation.param_env)
-                    .eq(target, new_tuple)
+                    .eq(DefineOpaqueTypes::No, target, new_tuple)
                     .map_err(|_| Unimplemented)?;
                 nested.extend(obligations);
 
-                // Construct the nested `T: Unsize<U>` predicate.
-                nested.push(ensure_sufficient_stack(|| {
-                    predicate_for_trait_def(
-                        tcx,
-                        obligation.param_env,
-                        obligation.cause.clone(),
-                        obligation.predicate.def_id(),
-                        obligation.recursion_depth + 1,
-                        [a_last, b_last],
-                    )
-                }));
+                // Add a nested `T: Unsize<U>` predicate.
+                let last_unsize_obligation = obligation
+                    .with(tcx, tcx.mk_trait_ref(obligation.predicate.def_id(), [a_last, b_last]));
+                nested.push(last_unsize_obligation);
             }
 
             _ => bug!("source: {source}, target: {target}"),
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index 7f454fbb3..6bb53418b 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -2,12 +2,6 @@
 //!
 //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/traits/resolution.html#selection
 
-// FIXME: The `map` field in ProvisionalEvaluationCache should be changed to
-// a `FxIndexMap` to avoid query instability, but right now it causes a perf regression. This would be
-// fixed or at least lightened by the addition of the `drain_filter` method to `FxIndexMap`
-// Relevant: https://github.com/rust-lang/rust/pull/103723 and https://github.com/bluss/indexmap/issues/242
-#![allow(rustc::potential_query_instability)]
-
 use self::EvaluationResult::*;
 use self::SelectionCandidate::*;
 
@@ -17,7 +11,7 @@ use super::project;
 use super::project::normalize_with_depth_to;
 use super::project::ProjectionTyObligation;
 use super::util;
-use super::util::{closure_trait_ref_and_return_type, predicate_for_trait_def};
+use super::util::closure_trait_ref_and_return_type;
 use super::wf;
 use super::{
     ErrorReporting, ImplDerivedObligation, ImplDerivedObligationCause, Normalized, Obligation,
@@ -32,25 +26,23 @@ use crate::traits::project::ProjectAndUnifyResult;
 use crate::traits::project::ProjectionCacheKeyExt;
 use crate::traits::ProjectionCacheKey;
 use crate::traits::Unimplemented;
-use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
+use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::Diagnostic;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
+use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::infer::LateBoundRegionConversionTime;
 use rustc_infer::traits::TraitEngine;
 use rustc_infer::traits::TraitEngineExt;
 use rustc_middle::dep_graph::{DepKind, DepNodeIndex};
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::abstract_const::NotConstEvaluatable;
-use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
 use rustc_middle::ty::fold::BottomUpFolder;
 use rustc_middle::ty::relate::TypeRelation;
 use rustc_middle::ty::SubstsRef;
 use rustc_middle::ty::{self, EarlyBinder, PolyProjectionPredicate, ToPolyTraitRef, ToPredicate};
 use rustc_middle::ty::{Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
-use rustc_session::config::TraitSolver;
 use rustc_span::symbol::sym;
 
 use std::cell::{Cell, RefCell};
@@ -178,14 +170,14 @@ struct TraitObligationStack<'prev, 'tcx> {
 }
 
 struct SelectionCandidateSet<'tcx> {
-    // A list of candidates that definitely apply to the current
-    // obligation (meaning: types unify).
+    /// A list of candidates that definitely apply to the current
+    /// obligation (meaning: types unify).
     vec: Vec<SelectionCandidate<'tcx>>,
 
-    // If `true`, then there were candidates that might or might
-    // not have applied, but we couldn't tell. This occurs when some
-    // of the input types are type variables, in which case there are
-    // various "builtin" rules that might or might not trigger.
+    /// If `true`, then there were candidates that might or might
+    /// not have applied, but we couldn't tell. This occurs when some
+    /// of the input types are type variables, in which case there are
+    /// various "builtin" rules that might or might not trigger.
     ambiguous: bool,
 }
 
@@ -211,7 +203,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     pub fn new(infcx: &'cx InferCtxt<'tcx>) -> SelectionContext<'cx, 'tcx> {
         SelectionContext {
             infcx,
-            freshener: infcx.freshener_keep_static(),
+            freshener: infcx.freshener(),
             intercrate_ambiguity_causes: None,
             query_mode: TraitQueryMode::Standard,
         }
@@ -465,14 +457,14 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         if candidates.len() > 1 {
             let mut i = 0;
             while i < candidates.len() {
-                let is_dup = (0..candidates.len()).filter(|&j| i != j).any(|j| {
+                let should_drop_i = (0..candidates.len()).filter(|&j| i != j).any(|j| {
                     self.candidate_should_be_dropped_in_favor_of(
                         &candidates[i],
                         &candidates[j],
                         needs_infer,
-                    )
+                    ) == DropVictim::Yes
                 });
-                if is_dup {
+                if should_drop_i {
                     debug!(candidate = ?candidates[i], "Dropping candidate #{}/{}", i, candidates.len());
                     candidates.swap_remove(i);
                 } else {
@@ -545,13 +537,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         obligation: &PredicateObligation<'tcx>,
     ) -> Result<EvaluationResult, OverflowError> {
         self.evaluation_probe(|this| {
-            if this.tcx().sess.opts.unstable_opts.trait_solver != TraitSolver::Next {
+            if this.tcx().trait_solver_next() {
+                this.evaluate_predicates_recursively_in_new_solver([obligation.clone()])
+            } else {
                 this.evaluate_predicate_recursively(
                     TraitObligationStackList::empty(&ProvisionalEvaluationCache::default()),
                     obligation.clone(),
                 )
-            } else {
-                this.evaluate_predicates_recursively_in_new_solver([obligation.clone()])
             }
         })
     }
@@ -591,9 +583,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     where
         I: IntoIterator<Item = PredicateObligation<'tcx>> + std::fmt::Debug,
     {
-        if self.tcx().sess.opts.unstable_opts.trait_solver != TraitSolver::Next {
+        if self.tcx().trait_solver_next() {
+            self.evaluate_predicates_recursively_in_new_solver(predicates)
+        } else {
             let mut result = EvaluatedToOk;
-            for obligation in predicates {
+            for mut obligation in predicates {
+                obligation.set_depth_from_parent(stack.depth());
                 let eval = self.evaluate_predicate_recursively(stack, obligation.clone())?;
                 if let EvaluatedToErr = eval {
                     // fast-path - EvaluatedToErr is the top of the lattice,
@@ -604,8 +599,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 }
             }
             Ok(result)
-        } else {
-            self.evaluate_predicates_recursively_in_new_solver(predicates)
         }
     }
 
@@ -617,6 +610,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let mut fulfill_cx = crate::solve::FulfillmentCtxt::new();
         fulfill_cx.register_predicate_obligations(self.infcx, predicates);
         // True errors
+        // FIXME(-Ztrait-solver=next): Overflows are reported as ambig here, is that OK?
         if !fulfill_cx.select_where_possible(self.infcx).is_empty() {
             return Ok(EvaluatedToErr);
         }
@@ -661,12 +655,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     let p = bound_predicate.rebind(p);
                     // Does this code ever run?
                     match self.infcx.subtype_predicate(&obligation.cause, obligation.param_env, p) {
-                        Ok(Ok(InferOk { mut obligations, .. })) => {
-                            self.add_depth(obligations.iter_mut(), obligation.recursion_depth);
-                            self.evaluate_predicates_recursively(
-                                previous_stack,
-                                obligations.into_iter(),
-                            )
+                        Ok(Ok(InferOk { obligations, .. })) => {
+                            self.evaluate_predicates_recursively(previous_stack, obligations)
                         }
                         Ok(Err(_)) => Ok(EvaluatedToErr),
                         Err(..) => Ok(EvaluatedToAmbig),
@@ -677,12 +667,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     let p = bound_predicate.rebind(p);
                     // Does this code ever run?
                     match self.infcx.coerce_predicate(&obligation.cause, obligation.param_env, p) {
-                        Ok(Ok(InferOk { mut obligations, .. })) => {
-                            self.add_depth(obligations.iter_mut(), obligation.recursion_depth);
-                            self.evaluate_predicates_recursively(
-                                previous_stack,
-                                obligations.into_iter(),
-                            )
+                        Ok(Ok(InferOk { obligations, .. })) => {
+                            self.evaluate_predicates_recursively(previous_stack, obligations)
                         }
                         Ok(Err(_)) => Ok(EvaluatedToErr),
                         Err(..) => Ok(EvaluatedToAmbig),
@@ -755,9 +741,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                         arg,
                         obligation.cause.span,
                     ) {
-                        Some(mut obligations) => {
-                            self.add_depth(obligations.iter_mut(), obligation.recursion_depth);
-
+                        Some(obligations) => {
                             cache.wf_args.borrow_mut().push((arg, previous_stack.depth()));
                             let result =
                                 self.evaluate_predicates_recursively(previous_stack, obligations);
@@ -778,14 +762,16 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 }
 
                 ty::PredicateKind::Clause(ty::Clause::TypeOutlives(pred)) => {
-                    // A global type with no late-bound regions can only
-                    // contain the "'static" lifetime (any other lifetime
-                    // would either be late-bound or local), so it is guaranteed
-                    // to outlive any other lifetime
-                    if pred.0.is_global() && !pred.0.has_late_bound_vars() {
-                        Ok(EvaluatedToOk)
-                    } else {
+                    // A global type with no free lifetimes or generic parameters
+                    // outlives anything.
+                    if pred.0.has_free_regions()
+                        || pred.0.has_late_bound_regions()
+                        || pred.0.has_non_region_infer()
+                        || pred.0.has_non_region_infer()
+                    {
                         Ok(EvaluatedToOkModuloRegions)
+                    } else {
+                        Ok(EvaluatedToOk)
                     }
                 }
 
@@ -826,10 +812,14 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                                     }
                                 }
 
-                                self.add_depth(
-                                    subobligations.iter_mut(),
-                                    obligation.recursion_depth,
-                                );
+                                // Need to explicitly set the depth of nested goals here as
+                                // projection obligations can cycle by themselves and in
+                                // `evaluate_predicates_recursively` we only add the depth
+                                // for parent trait goals because only these get added to the
+                                // `TraitObligationStackList`.
+                                for subobligation in subobligations.iter_mut() {
+                                    subobligation.set_depth_from_parent(obligation.recursion_depth);
+                                }
                                 let res = self.evaluate_predicates_recursively(
                                     previous_stack,
                                     subobligations,
@@ -909,38 +899,28 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                                 if a.def.did == b.def.did
                                     && tcx.def_kind(a.def.did) == DefKind::AssocConst =>
                             {
-                                if let Ok(new_obligations) = self
+                                if let Ok(InferOk { obligations, value: () }) = self
                                     .infcx
                                     .at(&obligation.cause, obligation.param_env)
                                     .trace(c1, c2)
-                                    .eq(a.substs, b.substs)
+                                    .eq(DefineOpaqueTypes::No, a.substs, b.substs)
                                 {
-                                    let mut obligations = new_obligations.obligations;
-                                    self.add_depth(
-                                        obligations.iter_mut(),
-                                        obligation.recursion_depth,
-                                    );
                                     return self.evaluate_predicates_recursively(
                                         previous_stack,
-                                        obligations.into_iter(),
+                                        obligations,
                                     );
                                 }
                             }
                             (_, Unevaluated(_)) | (Unevaluated(_), _) => (),
                             (_, _) => {
-                                if let Ok(new_obligations) = self
+                                if let Ok(InferOk { obligations, value: () }) = self
                                     .infcx
                                     .at(&obligation.cause, obligation.param_env)
-                                    .eq(c1, c2)
+                                    .eq(DefineOpaqueTypes::No, c1, c2)
                                 {
-                                    let mut obligations = new_obligations.obligations;
-                                    self.add_depth(
-                                        obligations.iter_mut(),
-                                        obligation.recursion_depth,
-                                    );
                                     return self.evaluate_predicates_recursively(
                                         previous_stack,
-                                        obligations.into_iter(),
+                                        obligations,
                                     );
                                 }
                             }
@@ -965,8 +945,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
                     match (evaluate(c1), evaluate(c2)) {
                         (Ok(c1), Ok(c2)) => {
-                            match self.infcx.at(&obligation.cause, obligation.param_env).eq(c1, c2)
-                            {
+                            match self.infcx.at(&obligation.cause, obligation.param_env).eq(
+                                DefineOpaqueTypes::No,
+                                c1,
+                                c2,
+                            ) {
                                 Ok(inf_ok) => self.evaluate_predicates_recursively(
                                     previous_stack,
                                     inf_ok.into_obligations(),
@@ -989,12 +972,16 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 ty::PredicateKind::TypeWellFormedFromEnv(..) => {
                     bug!("TypeWellFormedFromEnv is only used for chalk")
                 }
-                ty::PredicateKind::AliasEq(..) => {
-                    bug!("AliasEq is only used for new solver")
+                ty::PredicateKind::AliasRelate(..) => {
+                    bug!("AliasRelate is only used for new solver")
                 }
                 ty::PredicateKind::Ambiguous => Ok(EvaluatedToAmbig),
                 ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
-                    match self.infcx.at(&obligation.cause, obligation.param_env).eq(ct.ty(), ty) {
+                    match self.infcx.at(&obligation.cause, obligation.param_env).eq(
+                        DefineOpaqueTypes::No,
+                        ct.ty(),
+                        ty,
+                    ) {
                         Ok(inf_ok) => self.evaluate_predicates_recursively(
                             previous_stack,
                             inf_ok.into_obligations(),
@@ -1359,24 +1346,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         self.infcx.evaluation_cache.insert((param_env, trait_pred), dep_node, result);
     }
 
-    /// For various reasons, it's possible for a subobligation
-    /// to have a *lower* recursion_depth than the obligation used to create it.
-    /// Projection sub-obligations may be returned from the projection cache,
-    /// which results in obligations with an 'old' `recursion_depth`.
-    /// Additionally, methods like `InferCtxt.subtype_predicate` produce
-    /// subobligations without taking in a 'parent' depth, causing the
-    /// generated subobligations to have a `recursion_depth` of `0`.
-    ///
-    /// To ensure that obligation_depth never decreases, we force all subobligations
-    /// to have at least the depth of the original obligation.
-    fn add_depth<T: 'cx, I: Iterator<Item = &'cx mut Obligation<'tcx, T>>>(
-        &self,
-        it: I,
-        min_depth: usize,
-    ) {
-        it.for_each(|o| o.recursion_depth = cmp::max(min_depth, o.recursion_depth) + 1);
-    }
-
     fn check_recursion_depth<T>(
         &self,
         depth: usize,
@@ -1752,7 +1721,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         });
         self.infcx
             .at(&obligation.cause, obligation.param_env)
-            .sup(ty::Binder::dummy(placeholder_trait_ref), trait_bound)
+            .sup(DefineOpaqueTypes::No, ty::Binder::dummy(placeholder_trait_ref), trait_bound)
             .map(|InferOk { obligations: _, value: () }| {
                 // This method is called within a probe, so we can't have
                 // inference variables and placeholders escape.
@@ -1814,7 +1783,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let is_match = self
             .infcx
             .at(&obligation.cause, obligation.param_env)
-            .sup(obligation.predicate, infer_projection)
+            .sup(DefineOpaqueTypes::No, obligation.predicate, infer_projection)
             .map_or(false, |InferOk { obligations, value: () }| {
                 self.evaluate_predicates_recursively(
                     TraitObligationStackList::empty(&ProvisionalEvaluationCache::default()),
@@ -1842,16 +1811,28 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             ProjectionMatchesProjection::No
         }
     }
+}
 
-    ///////////////////////////////////////////////////////////////////////////
-    // WINNOW
-    //
-    // Winnowing is the process of attempting to resolve ambiguity by
-    // probing further. During the winnowing process, we unify all
-    // type variables and then we also attempt to evaluate recursive
-    // bounds to see if they are satisfied.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+enum DropVictim {
+    Yes,
+    No,
+}
+
+impl DropVictim {
+    fn drop_if(should_drop: bool) -> DropVictim {
+        if should_drop { DropVictim::Yes } else { DropVictim::No }
+    }
+}
 
-    /// Returns `true` if `victim` should be dropped in favor of
+/// ## Winnowing
+///
+/// Winnowing is the process of attempting to resolve ambiguity by
+/// probing further. During the winnowing process, we unify all
+/// type variables and then we also attempt to evaluate recursive
+/// bounds to see if they are satisfied.
+impl<'tcx> SelectionContext<'_, 'tcx> {
+    /// Returns `DropVictim::Yes` if `victim` should be dropped in favor of
     /// `other`. Generally speaking we will drop duplicate
     /// candidates and prefer where-clause candidates.
     ///
@@ -1861,9 +1842,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         victim: &EvaluatedCandidate<'tcx>,
         other: &EvaluatedCandidate<'tcx>,
         needs_infer: bool,
-    ) -> bool {
+    ) -> DropVictim {
         if victim.candidate == other.candidate {
-            return true;
+            return DropVictim::Yes;
         }
 
         // Check if a bound would previously have been removed when normalizing
@@ -1887,11 +1868,15 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             }
 
             // FIXME(@jswrenn): this should probably be more sophisticated
-            (TransmutabilityCandidate, _) | (_, TransmutabilityCandidate) => false,
+            (TransmutabilityCandidate, _) | (_, TransmutabilityCandidate) => DropVictim::No,
 
             // (*)
-            (BuiltinCandidate { has_nested: false } | ConstDestructCandidate(_), _) => true,
-            (_, BuiltinCandidate { has_nested: false } | ConstDestructCandidate(_)) => false,
+            (BuiltinCandidate { has_nested: false } | ConstDestructCandidate(_), _) => {
+                DropVictim::Yes
+            }
+            (_, BuiltinCandidate { has_nested: false } | ConstDestructCandidate(_)) => {
+                DropVictim::No
+            }
 
             (ParamCandidate(other), ParamCandidate(victim)) => {
                 let same_except_bound_vars = other.skip_binder().trait_ref
@@ -1905,28 +1890,23 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     // or the current one if tied (they should both evaluate to the same answer). This is
                     // probably best characterized as a "hack", since we might prefer to just do our
                     // best to *not* create essentially duplicate candidates in the first place.
-                    other.bound_vars().len() <= victim.bound_vars().len()
+                    DropVictim::drop_if(other.bound_vars().len() <= victim.bound_vars().len())
                 } else if other.skip_binder().trait_ref == victim.skip_binder().trait_ref
                     && victim.skip_binder().constness == ty::BoundConstness::NotConst
                     && other.skip_binder().polarity == victim.skip_binder().polarity
                 {
                     // Drop otherwise equivalent non-const candidates in favor of const candidates.
-                    true
+                    DropVictim::Yes
                 } else {
-                    false
+                    DropVictim::No
                 }
             }
 
             // Drop otherwise equivalent non-const fn pointer candidates
-            (FnPointerCandidate { .. }, FnPointerCandidate { is_const: false }) => true,
+            (FnPointerCandidate { .. }, FnPointerCandidate { is_const: false }) => DropVictim::Yes,
 
-            // Global bounds from the where clause should be ignored
-            // here (see issue #50825). Otherwise, we have a where
-            // clause so don't go around looking for impls.
-            // Arbitrarily give param candidates priority
-            // over projection and object candidates.
             (
-                ParamCandidate(ref cand),
+                ParamCandidate(ref other_cand),
                 ImplCandidate(..)
                 | ClosureCandidate { .. }
                 | GeneratorCandidate
@@ -1939,11 +1919,19 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 | TraitAliasCandidate
                 | ObjectCandidate(_)
                 | ProjectionCandidate(..),
-            ) => !is_global(cand),
-            (ObjectCandidate(_) | ProjectionCandidate(..), ParamCandidate(ref cand)) => {
+            ) => {
+                // We have a where clause so don't go around looking
+                // for impls. Arbitrarily give param candidates priority
+                // over projection and object candidates.
+                //
+                // Global bounds from the where clause should be ignored
+                // here (see issue #50825).
+                DropVictim::drop_if(!is_global(other_cand))
+            }
+            (ObjectCandidate(_) | ProjectionCandidate(..), ParamCandidate(ref victim_cand)) => {
                 // Prefer these to a global where-clause bound
                 // (see issue #50825).
-                is_global(cand)
+                if is_global(victim_cand) { DropVictim::Yes } else { DropVictim::No }
             }
             (
                 ImplCandidate(_)
@@ -1956,18 +1944,20 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 | TraitUpcastingUnsizeCandidate(_)
                 | BuiltinCandidate { has_nested: true }
                 | TraitAliasCandidate,
-                ParamCandidate(ref cand),
+                ParamCandidate(ref victim_cand),
             ) => {
                 // Prefer these to a global where-clause bound
                 // (see issue #50825).
-                is_global(cand) && other.evaluation.must_apply_modulo_regions()
+                DropVictim::drop_if(
+                    is_global(victim_cand) && other.evaluation.must_apply_modulo_regions(),
+                )
             }
 
             (ProjectionCandidate(i, _), ProjectionCandidate(j, _))
             | (ObjectCandidate(i), ObjectCandidate(j)) => {
                 // Arbitrarily pick the lower numbered candidate for backwards
                 // compatibility reasons. Don't let this affect inference.
-                i < j && !needs_infer
+                DropVictim::drop_if(i < j && !needs_infer)
             }
             (ObjectCandidate(_), ProjectionCandidate(..))
             | (ProjectionCandidate(..), ObjectCandidate(_)) => {
@@ -1987,7 +1977,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 | TraitUpcastingUnsizeCandidate(_)
                 | BuiltinCandidate { .. }
                 | TraitAliasCandidate,
-            ) => true,
+            ) => DropVictim::Yes,
 
             (
                 ImplCandidate(..)
@@ -2001,7 +1991,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 | BuiltinCandidate { .. }
                 | TraitAliasCandidate,
                 ObjectCandidate(_) | ProjectionCandidate(..),
-            ) => false,
+            ) => DropVictim::No,
 
             (&ImplCandidate(other_def), &ImplCandidate(victim_def)) => {
                 // See if we can toss out `victim` based on specialization.
@@ -2014,59 +2004,69 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 let tcx = self.tcx();
                 if other.evaluation.must_apply_modulo_regions() {
                     if tcx.specializes((other_def, victim_def)) {
-                        return true;
+                        return DropVictim::Yes;
                     }
                 }
 
-                if other.evaluation.must_apply_considering_regions() {
-                    match tcx.impls_are_allowed_to_overlap(other_def, victim_def) {
-                        Some(ty::ImplOverlapKind::Permitted { marker: true }) => {
-                            // Subtle: If the predicate we are evaluating has inference
-                            // variables, do *not* allow discarding candidates due to
-                            // marker trait impls.
-                            //
-                            // Without this restriction, we could end up accidentally
-                            // constraining inference variables based on an arbitrarily
-                            // chosen trait impl.
-                            //
-                            // Imagine we have the following code:
-                            //
-                            // ```rust
-                            // #[marker] trait MyTrait {}
-                            // impl MyTrait for u8 {}
-                            // impl MyTrait for bool {}
-                            // ```
-                            //
-                            // And we are evaluating the predicate `<_#0t as MyTrait>`.
-                            //
-                            // During selection, we will end up with one candidate for each
-                            // impl of `MyTrait`. If we were to discard one impl in favor
-                            // of the other, we would be left with one candidate, causing
-                            // us to "successfully" select the predicate, unifying
-                            // _#0t with (for example) `u8`.
-                            //
-                            // However, we have no reason to believe that this unification
-                            // is correct - we've essentially just picked an arbitrary
-                            // *possibility* for _#0t, and required that this be the *only*
-                            // possibility.
-                            //
-                            // Eventually, we will either:
-                            // 1) Unify all inference variables in the predicate through
-                            // some other means (e.g. type-checking of a function). We will
-                            // then be in a position to drop marker trait candidates
-                            // without constraining inference variables (since there are
-                            // none left to constrain)
-                            // 2) Be left with some unconstrained inference variables. We
-                            // will then correctly report an inference error, since the
-                            // existence of multiple marker trait impls tells us nothing
-                            // about which one should actually apply.
-                            !needs_infer
-                        }
-                        Some(_) => true,
-                        None => false,
+                match tcx.impls_are_allowed_to_overlap(other_def, victim_def) {
+                    // For #33140 the impl headers must be exactly equal, the trait must not have
+                    // any associated items and there are no where-clauses.
+                    //
+                    // We can just arbitrarily drop one of the impls.
+                    Some(ty::ImplOverlapKind::Issue33140) => {
+                        assert_eq!(other.evaluation, victim.evaluation);
+                        DropVictim::Yes
                     }
-                } else {
-                    false
+                    // For candidates which already reference errors it doesn't really
+                    // matter what we do 🤷
+                    Some(ty::ImplOverlapKind::Permitted { marker: false }) => {
+                        DropVictim::drop_if(other.evaluation.must_apply_considering_regions())
+                    }
+                    Some(ty::ImplOverlapKind::Permitted { marker: true }) => {
+                        // Subtle: If the predicate we are evaluating has inference
+                        // variables, do *not* allow discarding candidates due to
+                        // marker trait impls.
+                        //
+                        // Without this restriction, we could end up accidentally
+                        // constraining inference variables based on an arbitrarily
+                        // chosen trait impl.
+                        //
+                        // Imagine we have the following code:
+                        //
+                        // ```rust
+                        // #[marker] trait MyTrait {}
+                        // impl MyTrait for u8 {}
+                        // impl MyTrait for bool {}
+                        // ```
+                        //
+                        // And we are evaluating the predicate `<_#0t as MyTrait>`.
+                        //
+                        // During selection, we will end up with one candidate for each
+                        // impl of `MyTrait`. If we were to discard one impl in favor
+                        // of the other, we would be left with one candidate, causing
+                        // us to "successfully" select the predicate, unifying
+                        // _#0t with (for example) `u8`.
+                        //
+                        // However, we have no reason to believe that this unification
+                        // is correct - we've essentially just picked an arbitrary
+                        // *possibility* for _#0t, and required that this be the *only*
+                        // possibility.
+                        //
+                        // Eventually, we will either:
+                        // 1) Unify all inference variables in the predicate through
+                        // some other means (e.g. type-checking of a function). We will
+                        // then be in a position to drop marker trait candidates
+                        // without constraining inference variables (since there are
+                        // none left to constrain)
+                        // 2) Be left with some unconstrained inference variables. We
+                        // will then correctly report an inference error, since the
+                        // existence of multiple marker trait impls tells us nothing
+                        // about which one should actually apply.
+                        DropVictim::drop_if(
+                            !needs_infer && other.evaluation.must_apply_considering_regions(),
+                        )
+                    }
+                    None => DropVictim::No,
                 }
             }
 
@@ -2092,10 +2092,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 | TraitUpcastingUnsizeCandidate(_)
                 | BuiltinCandidate { has_nested: true }
                 | TraitAliasCandidate,
-            ) => false,
+            ) => DropVictim::No,
         }
     }
+}
 
+impl<'tcx> SelectionContext<'_, 'tcx> {
     fn sized_conditions(
         &mut self,
         obligation: &TraitObligation<'tcx>,
@@ -2149,7 +2151,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             ty::Alias(..) | ty::Param(_) | ty::Placeholder(..) => None,
             ty::Infer(ty::TyVar(_)) => Ambiguous,
 
-            // We can make this an ICE if/once we actually instantiate the trait obligation.
+            // We can make this an ICE if/once we actually instantiate the trait obligation eagerly.
             ty::Bound(..) => None,
 
             ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
@@ -2257,7 +2259,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 }
             }
 
-            ty::Adt(..) | ty::Alias(..) | ty::Param(..) => {
+            ty::Adt(..) | ty::Alias(..) | ty::Param(..) | ty::Placeholder(..) => {
                 // Fallback to whatever user-defined impls exist in this case.
                 None
             }
@@ -2269,9 +2271,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 Ambiguous
             }
 
-            ty::Placeholder(..)
-            | ty::Bound(..)
-            | ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
+            // We can make this an ICE if/once we actually instantiate the trait obligation eagerly.
+            ty::Bound(..) => None,
+
+            ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
                 bug!("asked to assemble builtin bounds of unexpected type: {:?}", self_ty);
             }
         }
@@ -2405,15 +2408,14 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                             placeholder_ty,
                         )
                     });
-                let placeholder_obligation = predicate_for_trait_def(
+
+                let obligation = Obligation::new(
                     self.tcx(),
-                    param_env,
                     cause.clone(),
-                    trait_def_id,
-                    recursion_depth,
-                    [normalized_ty],
+                    param_env,
+                    self.tcx().mk_trait_ref(trait_def_id, [normalized_ty]),
                 );
-                obligations.push(placeholder_obligation);
+                obligations.push(obligation);
                 obligations
             })
             .collect()
@@ -2507,7 +2509,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let InferOk { obligations, .. } = self
             .infcx
             .at(&cause, obligation.param_env)
-            .eq(placeholder_obligation_trait_ref, impl_trait_ref)
+            .eq(DefineOpaqueTypes::No, placeholder_obligation_trait_ref, impl_trait_ref)
             .map_err(|e| {
                 debug!("match_impl: failed eq_trait_refs due to `{}`", e.to_string(self.tcx()))
             })?;
@@ -2523,19 +2525,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         Ok(Normalized { value: impl_substs, obligations: nested_obligations })
     }
 
-    fn fast_reject_trait_refs(
-        &mut self,
-        obligation: &TraitObligation<'tcx>,
-        impl_trait_ref: &ty::TraitRef<'tcx>,
-    ) -> bool {
-        // We can avoid creating type variables and doing the full
-        // substitution if we find that any of the input types, when
-        // simplified, do not match.
-        let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsPlaceholder };
-        iter::zip(obligation.predicate.skip_binder().trait_ref.substs, impl_trait_ref.substs)
-            .any(|(obl, imp)| !drcx.generic_args_may_unify(obl, imp))
-    }
-
     /// Normalize `where_clause_trait_ref` and try to match it against
     /// `obligation`. If successful, return any predicates that
     /// result from the normalization.
@@ -2557,7 +2546,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     ) -> Result<Vec<PredicateObligation<'tcx>>, ()> {
         self.infcx
             .at(&obligation.cause, obligation.param_env)
-            .sup(obligation.predicate.to_poly_trait_ref(), poly_trait_ref)
+            .sup(DefineOpaqueTypes::No, obligation.predicate.to_poly_trait_ref(), poly_trait_ref)
             .map(|InferOk { obligations, .. }| obligations)
             .map_err(|_| ())
     }
@@ -2786,7 +2775,7 @@ struct ProvisionalEvaluationCache<'tcx> {
     /// - then we determine that `E` is in error -- we will then clear
     ///   all cache values whose DFN is >= 4 -- in this case, that
     ///   means the cached value for `F`.
-    map: RefCell<FxHashMap<ty::PolyTraitPredicate<'tcx>, ProvisionalEvaluation>>,
+    map: RefCell<FxIndexMap<ty::PolyTraitPredicate<'tcx>, ProvisionalEvaluation>>,
 
     /// The stack of args that we assume to be true because a `WF(arg)` predicate
     /// is on the stack above (and because of wellformedness is coinductive).
@@ -2934,12 +2923,13 @@ impl<'tcx> ProvisionalEvaluationCache<'tcx> {
     /// have a performance impact in practice.
     fn on_completion(&self, dfn: usize) {
         debug!(?dfn, "on_completion");
-
-        for (fresh_trait_pred, eval) in
-            self.map.borrow_mut().drain_filter(|_k, eval| eval.from_dfn >= dfn)
-        {
-            debug!(?fresh_trait_pred, ?eval, "on_completion");
-        }
+        self.map.borrow_mut().retain(|fresh_trait_pred, eval| {
+            if eval.from_dfn >= dfn {
+                debug!(?fresh_trait_pred, ?eval, "on_completion");
+                return false;
+            }
+            true
+        });
     }
 }
 
@@ -3021,7 +3011,7 @@ fn bind_generator_hidden_types_above<'tcx>(
                     if let ty::ReErased = r.kind() {
                         let br = ty::BoundRegion {
                             var: ty::BoundVar::from_u32(counter),
-                            kind: ty::BrAnon(counter, None),
+                            kind: ty::BrAnon(None),
                         };
                         counter += 1;
                         r = tcx.mk_re_late_bound(current_depth, br);
@@ -3037,7 +3027,7 @@ fn bind_generator_hidden_types_above<'tcx>(
         debug_assert!(!hidden_types.has_erased_regions());
     }
     let bound_vars = tcx.mk_bound_variable_kinds_from_iter(bound_vars.iter().chain(
-        (num_bound_variables..counter).map(|i| ty::BoundVariableKind::Region(ty::BrAnon(i, None))),
+        (num_bound_variables..counter).map(|_| ty::BoundVariableKind::Region(ty::BrAnon(None))),
     ));
     ty::Binder::bind_with_vars(hidden_types, bound_vars)
 }
diff --git a/compiler/rustc_trait_selection/src/traits/specialize/mod.rs b/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
index d1d6a7a90..8546bbe52 100644
--- a/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
@@ -10,6 +10,7 @@
 //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/traits/specialization.html
 
 pub mod specialization_graph;
+use rustc_infer::infer::DefineOpaqueTypes;
 use specialization_graph::GraphExt;
 
 use crate::errors::NegativePositiveConflict;
@@ -21,7 +22,7 @@ use crate::traits::{
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::{error_code, DelayDm, Diagnostic};
 use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_middle::ty::{self, ImplSubject, Ty, TyCtxt};
+use rustc_middle::ty::{self, ImplSubject, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::ty::{InternalSubsts, SubstsRef};
 use rustc_session::lint::builtin::COHERENCE_LEAK_CHECK;
 use rustc_session::lint::builtin::ORDER_DEPENDENT_TRAIT_OBJECTS;
@@ -99,10 +100,10 @@ pub fn translate_substs<'tcx>(
             }
 
             fulfill_implication(infcx, param_env, source_trait_ref, target_impl).unwrap_or_else(
-                |_| {
+                |()| {
                     bug!(
-                        "When translating substitutions for specialization, the expected \
-                         specialization failed to hold"
+                        "When translating substitutions from {source_impl:?} to {target_impl:?}, \
+                        the expected specialization failed to hold"
                     )
                 },
             )
@@ -193,7 +194,7 @@ fn fulfill_implication<'tcx>(
 
     // do the impls unify? If not, no specialization.
     let Ok(InferOk { obligations: more_obligations, .. }) =
-        infcx.at(&ObligationCause::dummy(), param_env).eq(source_trait, target_trait)
+        infcx.at(&ObligationCause::dummy(), param_env, ).eq(DefineOpaqueTypes::No,source_trait, target_trait)
     else {
         debug!(
             "fulfill_implication: {:?} does not unify with {:?}",
@@ -349,6 +350,10 @@ fn report_conflicting_impls<'tcx>(
         impl_span: Span,
         err: &mut Diagnostic,
     ) {
+        if (overlap.trait_ref, overlap.self_ty).references_error() {
+            err.downgrade_to_delayed_bug();
+        }
+
         match tcx.span_of_impl(overlap.with_impl) {
             Ok(span) => {
                 err.span_label(span, "first implementation here");
diff --git a/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs b/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs
index 61ed9ef2e..aa5c624f4 100644
--- a/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs
+++ b/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs
@@ -21,6 +21,7 @@ pub struct FutureCompatOverlapError<'tcx> {
 }
 
 /// The result of attempting to insert an impl into a group of children.
+#[derive(Debug)]
 enum Inserted<'tcx> {
     /// The impl was inserted as a new child in this group of children.
     BecameNewSibling(Option<FutureCompatOverlapError<'tcx>>),
@@ -49,7 +50,8 @@ impl<'tcx> ChildrenExt<'tcx> for Children {
     /// Insert an impl into this set of children without comparing to any existing impls.
     fn insert_blindly(&mut self, tcx: TyCtxt<'tcx>, impl_def_id: DefId) {
         let trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap().skip_binder();
-        if let Some(st) = fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::AsInfer)
+        if let Some(st) =
+            fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::AsCandidateKey)
         {
             debug!("insert_blindly: impl_def_id={:?} st={:?}", impl_def_id, st);
             self.non_blanket_impls.entry(st).or_default().push(impl_def_id)
@@ -65,7 +67,8 @@ impl<'tcx> ChildrenExt<'tcx> for Children {
     fn remove_existing(&mut self, tcx: TyCtxt<'tcx>, impl_def_id: DefId) {
         let trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap().skip_binder();
         let vec: &mut Vec<DefId>;
-        if let Some(st) = fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::AsInfer)
+        if let Some(st) =
+            fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::AsCandidateKey)
         {
             debug!("remove_existing: impl_def_id={:?} st={:?}", impl_def_id, st);
             vec = self.non_blanket_impls.get_mut(&st).unwrap();
@@ -80,6 +83,7 @@ impl<'tcx> ChildrenExt<'tcx> for Children {
 
     /// Attempt to insert an impl into this set of children, while comparing for
     /// specialization relationships.
+    #[instrument(level = "debug", skip(self, tcx), ret)]
     fn insert(
         &mut self,
         tcx: TyCtxt<'tcx>,
@@ -90,18 +94,13 @@ impl<'tcx> ChildrenExt<'tcx> for Children {
         let mut last_lint = None;
         let mut replace_children = Vec::new();
 
-        debug!("insert(impl_def_id={:?}, simplified_self={:?})", impl_def_id, simplified_self,);
-
         let possible_siblings = match simplified_self {
             Some(st) => PotentialSiblings::Filtered(filtered_children(self, st)),
             None => PotentialSiblings::Unfiltered(iter_children(self)),
         };
 
         for possible_sibling in possible_siblings {
-            debug!(
-                "insert: impl_def_id={:?}, simplified_self={:?}, possible_sibling={:?}",
-                impl_def_id, simplified_self, possible_sibling,
-            );
+            debug!(?possible_sibling);
 
             let create_overlap_error = |overlap: traits::coherence::OverlapResult<'tcx>| {
                 let trait_ref = overlap.impl_header.trait_ref.unwrap();
@@ -302,7 +301,8 @@ impl<'tcx> GraphExt<'tcx> for Graph {
 
         let mut parent = trait_def_id;
         let mut last_lint = None;
-        let simplified = fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::AsInfer);
+        let simplified =
+            fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::AsCandidateKey);
 
         // Descend the specialization tree, where `parent` is the current parent node.
         loop {
diff --git a/compiler/rustc_trait_selection/src/traits/util.rs b/compiler/rustc_trait_selection/src/traits/util.rs
index bcf63d5a6..20357d4d2 100644
--- a/compiler/rustc_trait_selection/src/traits/util.rs
+++ b/compiler/rustc_trait_selection/src/traits/util.rs
@@ -1,15 +1,14 @@
-use rustc_errors::Diagnostic;
-use rustc_span::Span;
-use smallvec::SmallVec;
-
+use super::NormalizeExt;
+use super::{ObligationCause, PredicateObligation, SelectionContext};
 use rustc_data_structures::fx::FxHashSet;
+use rustc_errors::Diagnostic;
 use rustc_hir::def_id::DefId;
+use rustc_infer::infer::InferOk;
+use rustc_middle::ty::SubstsRef;
 use rustc_middle::ty::{self, ImplSubject, ToPredicate, Ty, TyCtxt, TypeVisitableExt};
-use rustc_middle::ty::{GenericArg, SubstsRef};
+use rustc_span::Span;
+use smallvec::SmallVec;
 
-use super::NormalizeExt;
-use super::{Obligation, ObligationCause, PredicateObligation, SelectionContext};
-use rustc_infer::infer::InferOk;
 pub use rustc_infer::traits::{self, util::*};
 
 ///////////////////////////////////////////////////////////////////////////
@@ -116,7 +115,7 @@ impl<'tcx> TraitAliasExpander<'tcx> {
         }
 
         // Get components of trait alias.
-        let predicates = tcx.super_predicates_of(trait_ref.def_id());
+        let predicates = tcx.implied_predicates_of(trait_ref.def_id());
         debug!(?predicates);
 
         let items = predicates.predicates.iter().rev().filter_map(|(pred, span)| {
@@ -199,8 +198,9 @@ pub fn impl_subject_and_oblig<'a, 'tcx>(
     impl_def_id: DefId,
     impl_substs: SubstsRef<'tcx>,
 ) -> (ImplSubject<'tcx>, impl Iterator<Item = PredicateObligation<'tcx>>) {
-    let subject = selcx.tcx().bound_impl_subject(impl_def_id);
+    let subject = selcx.tcx().impl_subject(impl_def_id);
     let subject = subject.subst(selcx.tcx(), impl_substs);
+
     let InferOk { value: subject, obligations: normalization_obligations1 } =
         selcx.infcx.at(&ObligationCause::dummy(), param_env).normalize(subject);
 
@@ -218,33 +218,6 @@ pub fn impl_subject_and_oblig<'a, 'tcx>(
     (subject, impl_obligations)
 }
 
-pub fn predicate_for_trait_ref<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    cause: ObligationCause<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
-    trait_ref: ty::TraitRef<'tcx>,
-    recursion_depth: usize,
-) -> PredicateObligation<'tcx> {
-    Obligation {
-        cause,
-        param_env,
-        recursion_depth,
-        predicate: ty::Binder::dummy(trait_ref).without_const().to_predicate(tcx),
-    }
-}
-
-pub fn predicate_for_trait_def<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
-    cause: ObligationCause<'tcx>,
-    trait_def_id: DefId,
-    recursion_depth: usize,
-    params: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
-) -> PredicateObligation<'tcx> {
-    let trait_ref = tcx.mk_trait_ref(trait_def_id, params);
-    predicate_for_trait_ref(tcx, cause, param_env, trait_ref, recursion_depth)
-}
-
 /// Casts a trait reference into a reference to one of its super
 /// traits; returns `None` if `target_trait_def_id` is not a
 /// supertrait.
diff --git a/compiler/rustc_trait_selection/src/traits/wf.rs b/compiler/rustc_trait_selection/src/traits/wf.rs
index d498af359..3d026506a 100644
--- a/compiler/rustc_trait_selection/src/traits/wf.rs
+++ b/compiler/rustc_trait_selection/src/traits/wf.rs
@@ -191,8 +191,8 @@ pub fn predicate_obligations<'tcx>(
         ty::PredicateKind::TypeWellFormedFromEnv(..) => {
             bug!("TypeWellFormedFromEnv is only used for Chalk")
         }
-        ty::PredicateKind::AliasEq(..) => {
-            bug!("We should only wf check where clauses and `AliasEq` is not a `Clause`")
+        ty::PredicateKind::AliasRelate(..) => {
+            bug!("We should only wf check where clauses and `AliasRelate` is not a `Clause`")
         }
     }
 
@@ -364,7 +364,7 @@ impl<'tcx> WfPredicates<'tcx> {
         };
 
         if let Elaborate::All = elaborate {
-            let implied_obligations = traits::util::elaborate_obligations(tcx, obligations);
+            let implied_obligations = traits::util::elaborate(tcx, obligations);
             let implied_obligations = implied_obligations.map(extend);
             self.out.extend(implied_obligations);
         } else {
@@ -920,10 +920,10 @@ pub(crate) fn required_region_bounds<'tcx>(
 ) -> Vec<ty::Region<'tcx>> {
     assert!(!erased_self_ty.has_escaping_bound_vars());
 
-    traits::elaborate_predicates(tcx, predicates)
-        .filter_map(|obligation| {
-            debug!(?obligation);
-            match obligation.predicate.kind().skip_binder() {
+    traits::elaborate(tcx, predicates)
+        .filter_map(|pred| {
+            debug!(?pred);
+            match pred.kind().skip_binder() {
                 ty::PredicateKind::Clause(ty::Clause::Projection(..))
                 | ty::PredicateKind::Clause(ty::Clause::Trait(..))
                 | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
@@ -936,7 +936,7 @@ pub(crate) fn required_region_bounds<'tcx>(
                 | ty::PredicateKind::ConstEvaluatable(..)
                 | ty::PredicateKind::ConstEquate(..)
                 | ty::PredicateKind::Ambiguous
-                | ty::PredicateKind::AliasEq(..)
+                | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::TypeWellFormedFromEnv(..) => None,
                 ty::PredicateKind::Clause(ty::Clause::TypeOutlives(ty::OutlivesPredicate(
                     ref t,
diff --git a/compiler/rustc_traits/src/chalk/db.rs b/compiler/rustc_traits/src/chalk/db.rs
index f8c8f744e..9683e4847 100644
--- a/compiler/rustc_traits/src/chalk/db.rs
+++ b/compiler/rustc_traits/src/chalk/db.rs
@@ -730,7 +730,7 @@ fn bound_vars_for_item(tcx: TyCtxt<'_>, def_id: DefId) -> SubstsRef<'_> {
         ty::GenericParamDefKind::Lifetime => {
             let br = ty::BoundRegion {
                 var: ty::BoundVar::from_usize(substs.len()),
-                kind: ty::BrAnon(substs.len() as u32, None),
+                kind: ty::BrAnon(None),
             };
             tcx.mk_re_late_bound(ty::INNERMOST, br).into()
         }
diff --git a/compiler/rustc_traits/src/chalk/lowering.rs b/compiler/rustc_traits/src/chalk/lowering.rs
index 60e22d100..2be72879b 100644
--- a/compiler/rustc_traits/src/chalk/lowering.rs
+++ b/compiler/rustc_traits/src/chalk/lowering.rs
@@ -119,7 +119,7 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::InEnvironment<chalk_ir::Goal<RustInterner<'
                 },
                 ty::PredicateKind::ObjectSafe(..)
                 | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
-                | ty::PredicateKind::AliasEq(..)
+                | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::ClosureKind(..)
                 | ty::PredicateKind::Subtype(..)
                 | ty::PredicateKind::Coerce(..)
@@ -215,7 +215,7 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::GoalData<RustInterner<'tcx>>> for ty::Predi
             // some of these in terms of chalk operations.
             ty::PredicateKind::ClosureKind(..)
             | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
-            | ty::PredicateKind::AliasEq(..)
+            | ty::PredicateKind::AliasRelate(..)
             | ty::PredicateKind::Coerce(..)
             | ty::PredicateKind::ConstEvaluatable(..)
             | ty::PredicateKind::Ambiguous
@@ -376,7 +376,7 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Ty<RustInterner<'tcx>>> for Ty<'tcx> {
             ty::Placeholder(_placeholder) => {
                 chalk_ir::TyKind::Placeholder(chalk_ir::PlaceholderIndex {
                     ui: chalk_ir::UniverseIndex { counter: _placeholder.universe.as_usize() },
-                    idx: _placeholder.name.expect_anon() as usize,
+                    idx: _placeholder.bound.var.as_usize(),
                 })
             }
             ty::Infer(_infer) => unimplemented!(),
@@ -479,12 +479,15 @@ impl<'tcx> LowerInto<'tcx, Ty<'tcx>> for &chalk_ir::Ty<RustInterner<'tcx>> {
                 ty::DebruijnIndex::from_usize(bound.debruijn.depth() as usize),
                 ty::BoundTy {
                     var: ty::BoundVar::from_usize(bound.index),
-                    kind: ty::BoundTyKind::Anon(bound.index as u32),
+                    kind: ty::BoundTyKind::Anon,
                 },
             ),
             TyKind::Placeholder(placeholder) => ty::Placeholder(ty::Placeholder {
                 universe: ty::UniverseIndex::from_usize(placeholder.ui.counter),
-                name: ty::BoundTyKind::Anon(placeholder.idx as u32),
+                bound: ty::BoundTy {
+                    var: ty::BoundVar::from_usize(placeholder.idx),
+                    kind: ty::BoundTyKind::Anon,
+                },
             }),
             TyKind::InferenceVar(_, _) => unimplemented!(),
             TyKind::Dyn(_) => unimplemented!(),
@@ -530,13 +533,16 @@ impl<'tcx> LowerInto<'tcx, Region<'tcx>> for &chalk_ir::Lifetime<RustInterner<'t
                 ty::DebruijnIndex::from_u32(var.debruijn.depth()),
                 ty::BoundRegion {
                     var: ty::BoundVar::from_usize(var.index),
-                    kind: ty::BrAnon(var.index as u32, None),
+                    kind: ty::BrAnon(None),
                 },
             ),
             chalk_ir::LifetimeData::InferenceVar(_var) => unimplemented!(),
             chalk_ir::LifetimeData::Placeholder(p) => tcx.mk_re_placeholder(ty::Placeholder {
                 universe: ty::UniverseIndex::from_usize(p.ui.counter),
-                name: ty::BoundRegionKind::BrAnon(p.idx as u32, None),
+                bound: ty::BoundRegion {
+                    var: ty::BoundVar::from_usize(p.idx),
+                    kind: ty::BoundRegionKind::BrAnon(None),
+                },
             }),
             chalk_ir::LifetimeData::Static => tcx.lifetimes.re_static,
             chalk_ir::LifetimeData::Erased => tcx.lifetimes.re_erased,
@@ -652,7 +658,7 @@ impl<'tcx> LowerInto<'tcx, Option<chalk_ir::QuantifiedWhereClause<RustInterner<'
             ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..)) => None,
 
             ty::PredicateKind::ObjectSafe(..)
-            | ty::PredicateKind::AliasEq(..)
+            | ty::PredicateKind::AliasRelate(..)
             | ty::PredicateKind::ClosureKind(..)
             | ty::PredicateKind::Subtype(..)
             | ty::PredicateKind::Coerce(..)
@@ -685,7 +691,7 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Binders<chalk_ir::QuantifiedWhereClauses<Ru
         let self_ty = interner.tcx.mk_bound(
             // This is going to be wrapped in a binder
             ty::DebruijnIndex::from_usize(1),
-            ty::BoundTy { var: ty::BoundVar::from_usize(0), kind: ty::BoundTyKind::Anon(0) },
+            ty::BoundTy { var: ty::BoundVar::from_usize(0), kind: ty::BoundTyKind::Anon },
         );
         let where_clauses = predicates.into_iter().map(|predicate| {
             let (predicate, binders, _named_regions) =
@@ -787,7 +793,7 @@ impl<'tcx> LowerInto<'tcx, Option<chalk_solve::rust_ir::QuantifiedInlineBound<Ru
             ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..)) => None,
 
             ty::PredicateKind::Clause(ty::Clause::RegionOutlives(..))
-            | ty::PredicateKind::AliasEq(..)
+            | ty::PredicateKind::AliasRelate(..)
             | ty::PredicateKind::ObjectSafe(..)
             | ty::PredicateKind::ClosureKind(..)
             | ty::PredicateKind::Subtype(..)
@@ -909,8 +915,7 @@ pub(crate) fn collect_bound_vars<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
             .or_else(|| bug!("Skipped bound var index: parameters={:?}", parameters));
     });
 
-    let binders =
-        chalk_ir::VariableKinds::from_iter(interner, parameters.into_iter().map(|(_, v)| v));
+    let binders = chalk_ir::VariableKinds::from_iter(interner, parameters.into_values());
 
     (new_ty, binders, named_parameters)
 }
@@ -971,7 +976,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for BoundVarsCollector<'tcx> {
                     }
                 }
 
-                ty::BoundRegionKind::BrAnon(var, _) => match self.parameters.entry(var) {
+                ty::BoundRegionKind::BrAnon(_) => match self.parameters.entry(br.var.as_u32()) {
                     Entry::Vacant(entry) => {
                         entry.insert(chalk_ir::VariableKind::Lifetime);
                     }
@@ -1031,8 +1036,8 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for NamedBoundVarSubstitutor<'a, 'tcx> {
         match *r {
             ty::ReLateBound(index, br) if index == self.binder_index => match br.kind {
                 ty::BrNamed(def_id, _name) => match self.named_parameters.get(&def_id) {
-                    Some(idx) => {
-                        let new_br = ty::BoundRegion { var: br.var, kind: ty::BrAnon(*idx, None) };
+                    Some(_) => {
+                        let new_br = ty::BoundRegion { var: br.var, kind: ty::BrAnon(None) };
                         return self.tcx.mk_re_late_bound(index, new_br);
                     }
                     None => panic!("Missing `BrNamed`."),
@@ -1091,7 +1096,10 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ParamsSubstitutor<'tcx> {
             ty::Param(param) => match self.list.iter().position(|r| r == &param) {
                 Some(idx) => self.tcx.mk_placeholder(ty::PlaceholderType {
                     universe: ty::UniverseIndex::from_usize(0),
-                    name: ty::BoundTyKind::Anon(idx as u32),
+                    bound: ty::BoundTy {
+                        var: ty::BoundVar::from_usize(idx),
+                        kind: ty::BoundTyKind::Anon,
+                    },
                 }),
                 None => {
                     self.list.push(param);
@@ -1099,7 +1107,10 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ParamsSubstitutor<'tcx> {
                     self.params.insert(idx as u32, param);
                     self.tcx.mk_placeholder(ty::PlaceholderType {
                         universe: ty::UniverseIndex::from_usize(0),
-                        name: ty::BoundTyKind::Anon(idx as u32),
+                        bound: ty::BoundTy {
+                            var: ty::BoundVar::from_usize(idx),
+                            kind: ty::BoundTyKind::Anon,
+                        },
                     })
                 }
             },
@@ -1116,7 +1127,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ParamsSubstitutor<'tcx> {
                 Some(idx) => {
                     let br = ty::BoundRegion {
                         var: ty::BoundVar::from_u32(*idx),
-                        kind: ty::BrAnon(*idx, None),
+                        kind: ty::BrAnon(None),
                     };
                     self.tcx.mk_re_late_bound(self.binder_index, br)
                 }
@@ -1124,7 +1135,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ParamsSubstitutor<'tcx> {
                     let idx = self.named_regions.len() as u32;
                     let br = ty::BoundRegion {
                         var: ty::BoundVar::from_u32(idx),
-                        kind: ty::BrAnon(idx, None),
+                        kind: ty::BrAnon(None),
                     };
                     self.named_regions.insert(_re.def_id, idx);
                     self.tcx.mk_re_late_bound(self.binder_index, br)
@@ -1157,8 +1168,8 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseParamsSubstitutor<'tcx> {
 
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
         match *t.kind() {
-            ty::Placeholder(ty::PlaceholderType { universe: ty::UniverseIndex::ROOT, name }) => {
-                match self.params.get(&name.expect_anon()) {
+            ty::Placeholder(ty::PlaceholderType { universe: ty::UniverseIndex::ROOT, bound }) => {
+                match self.params.get(&bound.var.as_u32()) {
                     Some(&ty::ParamTy { index, name }) => self.tcx.mk_ty_param(index, name),
                     None => t,
                 }
@@ -1190,8 +1201,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for PlaceholdersCollector {
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
         match t.kind() {
             ty::Placeholder(p) if p.universe == self.universe_index => {
-                self.next_ty_placeholder =
-                    self.next_ty_placeholder.max(p.name.expect_anon() as usize + 1);
+                self.next_ty_placeholder = self.next_ty_placeholder.max(p.bound.var.as_usize() + 1);
             }
 
             _ => (),
@@ -1203,8 +1213,9 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for PlaceholdersCollector {
     fn visit_region(&mut self, r: Region<'tcx>) -> ControlFlow<Self::BreakTy> {
         match *r {
             ty::RePlaceholder(p) if p.universe == self.universe_index => {
-                if let ty::BoundRegionKind::BrAnon(anon, _) = p.name {
-                    self.next_anon_region_placeholder = self.next_anon_region_placeholder.max(anon);
+                if let ty::BoundRegionKind::BrAnon(_) = p.bound.kind {
+                    self.next_anon_region_placeholder =
+                        self.next_anon_region_placeholder.max(p.bound.var.as_u32());
                 }
                 // FIXME: This doesn't seem to handle BrNamed at all?
             }
diff --git a/compiler/rustc_traits/src/dropck_outlives.rs b/compiler/rustc_traits/src/dropck_outlives.rs
index b5924e949..58117c46f 100644
--- a/compiler/rustc_traits/src/dropck_outlives.rs
+++ b/compiler/rustc_traits/src/dropck_outlives.rs
@@ -292,7 +292,9 @@ pub(crate) fn adt_dtorck_constraint(
     let span = tcx.def_span(def_id);
     debug!("dtorck_constraint: {:?}", def);
 
-    if def.is_phantom_data() {
+    if def.is_manually_drop() {
+        bug!("`ManuallyDrop` should have been handled by `trivial_dropck_outlives`");
+    } else if def.is_phantom_data() {
         // The first generic parameter here is guaranteed to be a type because it's
         // `PhantomData`.
         let substs = InternalSubsts::identity_for_item(tcx, def_id);
diff --git a/compiler/rustc_traits/src/implied_outlives_bounds.rs b/compiler/rustc_traits/src/implied_outlives_bounds.rs
index ddd4ca143..f5bba14d2 100644
--- a/compiler/rustc_traits/src/implied_outlives_bounds.rs
+++ b/compiler/rustc_traits/src/implied_outlives_bounds.rs
@@ -86,7 +86,7 @@ fn compute_implied_outlives_bounds<'tcx>(
             if obligation.predicate.has_non_region_infer() {
                 match obligation.predicate.kind().skip_binder() {
                     ty::PredicateKind::Clause(ty::Clause::Projection(..))
-                    | ty::PredicateKind::AliasEq(..) => {
+                    | ty::PredicateKind::AliasRelate(..) => {
                         ocx.register_obligation(obligation.clone());
                     }
                     _ => {}
@@ -110,7 +110,7 @@ fn compute_implied_outlives_bounds<'tcx>(
                 | ty::PredicateKind::ConstEvaluatable(..)
                 | ty::PredicateKind::ConstEquate(..)
                 | ty::PredicateKind::Ambiguous
-                | ty::PredicateKind::AliasEq(..)
+                | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::TypeWellFormedFromEnv(..) => {}
 
                 // We need to search through *all* WellFormed predicates
diff --git a/compiler/rustc_traits/src/normalize_erasing_regions.rs b/compiler/rustc_traits/src/normalize_erasing_regions.rs
index f0597f192..126a494f3 100644
--- a/compiler/rustc_traits/src/normalize_erasing_regions.rs
+++ b/compiler/rustc_traits/src/normalize_erasing_regions.rs
@@ -61,7 +61,7 @@ fn not_outlives_predicate(p: ty::Predicate<'_>) -> bool {
         ty::PredicateKind::Clause(ty::Clause::Trait(..))
         | ty::PredicateKind::Clause(ty::Clause::Projection(..))
         | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
-        | ty::PredicateKind::AliasEq(..)
+        | ty::PredicateKind::AliasRelate(..)
         | ty::PredicateKind::WellFormed(..)
         | ty::PredicateKind::ObjectSafe(..)
         | ty::PredicateKind::ClosureKind(..)
diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs
index 295b65c2c..2a89494c8 100644
--- a/compiler/rustc_transmute/src/layout/tree.rs
+++ b/compiler/rustc_transmute/src/layout/tree.rs
@@ -167,31 +167,31 @@ where
     }
 }
 
-#[derive(Debug, Copy, Clone)]
-pub(crate) enum Err {
-    /// The layout of the type is unspecified.
-    Unspecified,
-    /// This error will be surfaced elsewhere by rustc, so don't surface it.
-    Unknown,
-}
-
 #[cfg(feature = "rustc")]
 pub(crate) mod rustc {
-    use super::{Err, Tree};
+    use super::Tree;
     use crate::layout::rustc::{Def, Ref};
 
-    use rustc_middle::ty;
     use rustc_middle::ty::layout::LayoutError;
     use rustc_middle::ty::util::Discr;
     use rustc_middle::ty::AdtDef;
     use rustc_middle::ty::ParamEnv;
     use rustc_middle::ty::SubstsRef;
-    use rustc_middle::ty::Ty;
-    use rustc_middle::ty::TyCtxt;
     use rustc_middle::ty::VariantDef;
+    use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
+    use rustc_span::ErrorGuaranteed;
     use rustc_target::abi::Align;
     use std::alloc;
 
+    #[derive(Debug, Copy, Clone)]
+    pub(crate) enum Err {
+        /// The layout of the type is unspecified.
+        Unspecified,
+        /// This error will be surfaced elsewhere by rustc, so don't surface it.
+        Unknown,
+        TypeError(ErrorGuaranteed),
+    }
+
     impl<'tcx> From<LayoutError<'tcx>> for Err {
         fn from(err: LayoutError<'tcx>) -> Self {
             match err {
@@ -261,6 +261,10 @@ pub(crate) mod rustc {
             use rustc_middle::ty::UintTy::*;
             use rustc_target::abi::HasDataLayout;
 
+            if let Err(e) = ty.error_reported() {
+                return Err(Err::TypeError(e));
+            }
+
             let target = tcx.data_layout();
 
             match ty.kind() {
diff --git a/compiler/rustc_transmute/src/lib.rs b/compiler/rustc_transmute/src/lib.rs
index 4b4a8ebd0..a93a42987 100644
--- a/compiler/rustc_transmute/src/lib.rs
+++ b/compiler/rustc_transmute/src/lib.rs
@@ -149,7 +149,7 @@ mod rustc {
                     .iter()
                     .enumerate()
                     .find(|(_, field_def)| name == field_def.name)
-                    .expect(&format!("There were no fields named `{name}`."));
+                    .unwrap_or_else(|| panic!("There were no fields named `{name}`."));
                 fields[field_idx].unwrap_leaf() == ScalarInt::TRUE
             };
 
diff --git a/compiler/rustc_transmute/src/maybe_transmutable/mod.rs b/compiler/rustc_transmute/src/maybe_transmutable/mod.rs
index 1186eac37..2e2fb90e7 100644
--- a/compiler/rustc_transmute/src/maybe_transmutable/mod.rs
+++ b/compiler/rustc_transmute/src/maybe_transmutable/mod.rs
@@ -56,7 +56,7 @@ where
 #[cfg(feature = "rustc")]
 mod rustc {
     use super::*;
-    use crate::layout::tree::Err;
+    use crate::layout::tree::rustc::Err;
 
     use rustc_middle::ty::Ty;
     use rustc_middle::ty::TyCtxt;
@@ -71,19 +71,20 @@ mod rustc {
                 // representations. If these conversions fail, conclude that the transmutation is
                 // unacceptable; the layouts of both the source and destination types must be
                 // well-defined.
-                let src = Tree::from_ty(src, context).map_err(|err| match err {
-                    // Answer `Yes` here, because "Unknown Type" will already be reported by
-                    // rustc. No need to spam the user with more errors.
-                    Err::Unknown => Answer::Yes,
-                    Err::Unspecified => Answer::No(Reason::SrcIsUnspecified),
-                })?;
+                let src = Tree::from_ty(src, context);
+                let dst = Tree::from_ty(dst, context);
 
-                let dst = Tree::from_ty(dst, context).map_err(|err| match err {
-                    Err::Unknown => Answer::Yes,
-                    Err::Unspecified => Answer::No(Reason::DstIsUnspecified),
-                })?;
-
-                Ok((src, dst))
+                match (src, dst) {
+                    // Answer `Yes` here, because 'unknown layout' and type errors will already
+                    // be reported by rustc. No need to spam the user with more errors.
+                    (Err(Err::TypeError(_)), _) => Err(Answer::Yes),
+                    (_, Err(Err::TypeError(_))) => Err(Answer::Yes),
+                    (Err(Err::Unknown), _) => Err(Answer::Yes),
+                    (_, Err(Err::Unknown)) => Err(Answer::Yes),
+                    (Err(Err::Unspecified), _) => Err(Answer::No(Reason::SrcIsUnspecified)),
+                    (_, Err(Err::Unspecified)) => Err(Answer::No(Reason::DstIsUnspecified)),
+                    (Ok(src), Ok(dst)) => Ok((src, dst)),
+                }
             });
 
             match query_or_answer {
diff --git a/compiler/rustc_transmute/src/maybe_transmutable/query_context.rs b/compiler/rustc_transmute/src/maybe_transmutable/query_context.rs
index e4f3e7928..0cae0377e 100644
--- a/compiler/rustc_transmute/src/maybe_transmutable/query_context.rs
+++ b/compiler/rustc_transmute/src/maybe_transmutable/query_context.rs
@@ -58,9 +58,7 @@ mod rustc {
             use rustc_middle::ty;
 
             let parent = if let ty::Adt(adt_def, ..) = scope.kind() {
-                use rustc_middle::ty::DefIdTree;
-                let parent = self.parent(adt_def.did());
-                parent
+                self.parent(adt_def.did())
             } else {
                 // Is this always how we want to handle a non-ADT scope?
                 return false;
diff --git a/compiler/rustc_transmute/src/maybe_transmutable/tests.rs b/compiler/rustc_transmute/src/maybe_transmutable/tests.rs
index 4d5772a4f..a8675f4ae 100644
--- a/compiler/rustc_transmute/src/maybe_transmutable/tests.rs
+++ b/compiler/rustc_transmute/src/maybe_transmutable/tests.rs
@@ -1,6 +1,6 @@
 use super::query_context::test::{Def, UltraMinimal};
 use crate::maybe_transmutable::MaybeTransmutableQuery;
-use crate::{layout, Answer, Reason, Set};
+use crate::{layout, Answer, Reason};
 use itertools::Itertools;
 
 mod bool {
@@ -48,9 +48,9 @@ mod bool {
 
         let into_set = |alts: Vec<_>| {
             #[cfg(feature = "rustc")]
-            let mut set = Set::default();
+            let mut set = crate::Set::default();
             #[cfg(not(feature = "rustc"))]
-            let mut set = Set::new();
+            let mut set = std::collections::HashSet::new();
             set.extend(alts);
             set
         };
diff --git a/compiler/rustc_ty_utils/locales/en-US.ftl b/compiler/rustc_ty_utils/messages.ftl
index abe65a0e3..15a14112f 100644
--- a/compiler/rustc_ty_utils/locales/en-US.ftl
+++ b/compiler/rustc_ty_utils/messages.ftl
@@ -12,7 +12,7 @@ ty_utils_array_not_supported = array construction is not supported in generic co
 
 ty_utils_block_not_supported = blocks are not supported in generic constants
 
-ty_utils_never_to_any_not_supported = converting nevers to any is not supported in generic constants
+ty_utils_never_to_any_not_supported = coercing the `never` type is not supported in generic constants
 
 ty_utils_tuple_not_supported = tuple construction is not supported in generic constants
 
@@ -45,3 +45,13 @@ ty_utils_control_flow_not_supported = control flow is not supported in generic c
 ty_utils_inline_asm_not_supported = assembly is not supported in generic constants
 
 ty_utils_operation_not_supported = unsupported operation in generic constants
+
+ty_utils_unexpected_fnptr_associated_item = `FnPtr` trait with unexpected associated item
+
+ty_utils_zero_length_simd_type = monomorphising SIMD type `{$ty}` of zero length
+
+ty_utils_multiple_array_fields_simd_type = monomorphising SIMD type `{$ty}` with more than one array field
+
+ty_utils_oversized_simd_type = monomorphising SIMD type `{$ty}` of length greater than {$max_lanes}
+
+ty_utils_non_primitive_simd_type = monomorphising SIMD type `{$ty}` with a non-primitive-scalar (integer/float/pointer) element type `{$e_ty}`
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index 35c9f95eb..271284b2d 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -3,7 +3,7 @@ use rustc_hir::lang_items::LangItem;
 use rustc_middle::ty::layout::{
     fn_can_unwind, FnAbiError, HasParamEnv, HasTyCtxt, LayoutCx, LayoutOf, TyAndLayout,
 };
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, InstanceDef, Ty, TyCtxt};
 use rustc_session::config::OptLevel;
 use rustc_span::def_id::DefId;
 use rustc_target::abi::call::{
@@ -29,6 +29,16 @@ fn fn_sig_for_fn_abi<'tcx>(
     instance: ty::Instance<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
 ) -> ty::PolyFnSig<'tcx> {
+    if let InstanceDef::ThreadLocalShim(..) = instance.def {
+        return ty::Binder::dummy(tcx.mk_fn_sig(
+            [],
+            tcx.thread_local_ptr_ty(instance.def_id()),
+            false,
+            hir::Unsafety::Normal,
+            rustc_target::spec::abi::Abi::Unadjusted,
+        ));
+    }
+
     let ty = instance.ty(tcx, param_env);
     match *ty.kind() {
         ty::FnDef(..) => {
@@ -539,7 +549,7 @@ fn make_thin_self_ptr<'tcx>(
         // get a built-in pointer type
         let mut fat_pointer_layout = layout;
         'descend_newtypes: while !fat_pointer_layout.ty.is_unsafe_ptr()
-            && !fat_pointer_layout.ty.is_region_ptr()
+            && !fat_pointer_layout.ty.is_ref()
         {
             for i in 0..fat_pointer_layout.fields.count() {
                 let field_layout = fat_pointer_layout.field(cx, i);
diff --git a/compiler/rustc_ty_utils/src/assoc.rs b/compiler/rustc_ty_utils/src/assoc.rs
index 0648784b2..de1e1a527 100644
--- a/compiler/rustc_ty_utils/src/assoc.rs
+++ b/compiler/rustc_ty_utils/src/assoc.rs
@@ -1,10 +1,10 @@
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
-use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::def_id::{DefId, DefIdMap, LocalDefId};
 use rustc_hir::definitions::DefPathData;
 use rustc_hir::intravisit::{self, Visitor};
-use rustc_middle::ty::{self, DefIdTree, ImplTraitInTraitData, InternalSubsts, TyCtxt};
+use rustc_middle::ty::{self, ImplTraitInTraitData, InternalSubsts, TyCtxt};
 use rustc_span::symbol::kw;
 
 pub fn provide(providers: &mut ty::query::Providers) {
@@ -12,20 +12,20 @@ pub fn provide(providers: &mut ty::query::Providers) {
         associated_item,
         associated_item_def_ids,
         associated_items,
-        associated_items_for_impl_trait_in_trait,
-        associated_item_for_impl_trait_in_trait,
+        associated_types_for_impl_traits_in_associated_fn,
+        associated_type_for_impl_trait_in_trait,
         impl_item_implementor_ids,
         ..*providers
     };
 }
 
-fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: DefId) -> &[DefId] {
-    let item = tcx.hir().expect_item(def_id.expect_local());
+fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &[DefId] {
+    let item = tcx.hir().expect_item(def_id);
     match item.kind {
         hir::ItemKind::Trait(.., ref trait_item_refs) => {
-            if tcx.sess.opts.unstable_opts.lower_impl_trait_in_trait_to_assoc_ty {
+            if tcx.lower_impl_trait_in_trait_to_assoc_ty() {
                 // We collect RPITITs for each trait method's return type and create a
-                // corresponding associated item using associated_items_for_impl_trait_in_trait
+                // corresponding associated item using associated_types_for_impl_traits_in_associated_fn
                 // query.
                 tcx.arena.alloc_from_iter(
                     trait_item_refs
@@ -40,7 +40,9 @@ fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: DefId) -> &[DefId] {
                                 .flat_map(|trait_item_ref| {
                                     let trait_fn_def_id =
                                         trait_item_ref.id.owner_id.def_id.to_def_id();
-                                    tcx.associated_items_for_impl_trait_in_trait(trait_fn_def_id)
+                                    tcx.associated_types_for_impl_traits_in_associated_fn(
+                                        trait_fn_def_id,
+                                    )
                                 })
                                 .map(|def_id| *def_id),
                         ),
@@ -54,10 +56,10 @@ fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: DefId) -> &[DefId] {
             }
         }
         hir::ItemKind::Impl(ref impl_) => {
-            if tcx.sess.opts.unstable_opts.lower_impl_trait_in_trait_to_assoc_ty {
+            if tcx.lower_impl_trait_in_trait_to_assoc_ty() {
                 // We collect RPITITs for each trait method's return type, on the impl side too and
                 // create a corresponding associated item using
-                // associated_items_for_impl_trait_in_trait query.
+                // associated_types_for_impl_traits_in_associated_fn query.
                 tcx.arena.alloc_from_iter(
                     impl_
                         .items
@@ -73,7 +75,9 @@ fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: DefId) -> &[DefId] {
                                 .flat_map(|impl_item_ref| {
                                     let impl_fn_def_id =
                                         impl_item_ref.id.owner_id.def_id.to_def_id();
-                                    tcx.associated_items_for_impl_trait_in_trait(impl_fn_def_id)
+                                    tcx.associated_types_for_impl_traits_in_associated_fn(
+                                        impl_fn_def_id,
+                                    )
                                 })
                                 .map(|def_id| *def_id)
                         })),
@@ -97,34 +101,33 @@ fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssocItems {
     }
 }
 
-fn impl_item_implementor_ids(tcx: TyCtxt<'_>, impl_id: DefId) -> FxHashMap<DefId, DefId> {
+fn impl_item_implementor_ids(tcx: TyCtxt<'_>, impl_id: DefId) -> DefIdMap<DefId> {
     tcx.associated_items(impl_id)
         .in_definition_order()
         .filter_map(|item| item.trait_item_def_id.map(|trait_item| (trait_item, item.def_id)))
         .collect()
 }
 
-fn associated_item(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssocItem {
-    let id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+fn associated_item(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::AssocItem {
+    let id = tcx.hir().local_def_id_to_hir_id(def_id);
     let parent_def_id = tcx.hir().get_parent_item(id);
     let parent_item = tcx.hir().expect_item(parent_def_id.def_id);
     match parent_item.kind {
         hir::ItemKind::Impl(ref impl_) => {
-            if let Some(impl_item_ref) =
-                impl_.items.iter().find(|i| i.id.owner_id.to_def_id() == def_id)
+            if let Some(impl_item_ref) = impl_.items.iter().find(|i| i.id.owner_id.def_id == def_id)
             {
                 let assoc_item = associated_item_from_impl_item_ref(impl_item_ref);
-                debug_assert_eq!(assoc_item.def_id, def_id);
+                debug_assert_eq!(assoc_item.def_id.expect_local(), def_id);
                 return assoc_item;
             }
         }
 
         hir::ItemKind::Trait(.., ref trait_item_refs) => {
             if let Some(trait_item_ref) =
-                trait_item_refs.iter().find(|i| i.id.owner_id.to_def_id() == def_id)
+                trait_item_refs.iter().find(|i| i.id.owner_id.def_id == def_id)
             {
                 let assoc_item = associated_item_from_trait_item_ref(trait_item_ref);
-                debug_assert_eq!(assoc_item.def_id, def_id);
+                debug_assert_eq!(assoc_item.def_id.expect_local(), def_id);
                 return assoc_item;
             }
         }
@@ -154,6 +157,7 @@ fn associated_item_from_trait_item_ref(trait_item_ref: &hir::TraitItemRef) -> ty
         trait_item_def_id: Some(owner_id.to_def_id()),
         container: ty::TraitContainer,
         fn_has_self_parameter: has_self,
+        opt_rpitit_info: None,
     }
 }
 
@@ -172,40 +176,53 @@ fn associated_item_from_impl_item_ref(impl_item_ref: &hir::ImplItemRef) -> ty::A
         trait_item_def_id: impl_item_ref.trait_item_def_id,
         container: ty::ImplContainer,
         fn_has_self_parameter: has_self,
+        opt_rpitit_info: None,
     }
 }
 
-/// Given an `fn_def_id` of a trait or of an impl that implements a given trait:
-/// if `fn_def_id` is the def id of a function defined inside a trait, then it creates and returns
-/// the associated items that correspond to each impl trait in return position for that trait.
-/// if `fn_def_id` is the def id of a function defined inside an impl that implements a trait, then it
-/// creates and returns the associated items that correspond to each impl trait in return position
-/// of the implemented trait.
-fn associated_items_for_impl_trait_in_trait(tcx: TyCtxt<'_>, fn_def_id: DefId) -> &'_ [DefId] {
-    let parent_def_id = tcx.parent(fn_def_id);
+/// Given an `fn_def_id` of a trait or a trait implementation:
+///
+/// if `fn_def_id` is a function defined inside a trait, then it synthesizes
+/// a new def id corresponding to a new associated type for each return-
+/// position `impl Trait` in the signature.
+///
+/// if `fn_def_id` is a function inside of an impl, then for each synthetic
+/// associated type generated for the corresponding trait function described
+/// above, synthesize a corresponding associated type in the impl.
+fn associated_types_for_impl_traits_in_associated_fn(
+    tcx: TyCtxt<'_>,
+    fn_def_id: LocalDefId,
+) -> &'_ [DefId] {
+    let parent_def_id = tcx.local_parent(fn_def_id);
 
     match tcx.def_kind(parent_def_id) {
         DefKind::Trait => {
-            struct RPITVisitor {
-                rpits: Vec<LocalDefId>,
+            struct RPITVisitor<'tcx> {
+                rpits: FxIndexSet<LocalDefId>,
+                tcx: TyCtxt<'tcx>,
             }
 
-            impl<'v> Visitor<'v> for RPITVisitor {
-                fn visit_ty(&mut self, ty: &'v hir::Ty<'v>) {
-                    if let hir::TyKind::OpaqueDef(item_id, _, _) = ty.kind {
-                        self.rpits.push(item_id.owner_id.def_id)
+            impl<'tcx> Visitor<'tcx> for RPITVisitor<'tcx> {
+                fn visit_ty(&mut self, ty: &'tcx hir::Ty<'tcx>) {
+                    if let hir::TyKind::OpaqueDef(item_id, _, _) = ty.kind
+                        && self.rpits.insert(item_id.owner_id.def_id)
+                    {
+                        let opaque_item = self.tcx.hir().expect_item(item_id.owner_id.def_id).expect_opaque_ty();
+                        for bound in opaque_item.bounds {
+                            intravisit::walk_param_bound(self, bound);
+                        }
                     }
                     intravisit::walk_ty(self, ty)
                 }
             }
 
-            let mut visitor = RPITVisitor { rpits: Vec::new() };
+            let mut visitor = RPITVisitor { tcx, rpits: FxIndexSet::default() };
 
-            if let Some(output) = tcx.hir().get_fn_output(fn_def_id.expect_local()) {
+            if let Some(output) = tcx.hir().get_fn_output(fn_def_id) {
                 visitor.visit_fn_ret_ty(output);
 
                 tcx.arena.alloc_from_iter(visitor.rpits.iter().map(|opaque_ty_def_id| {
-                    tcx.associated_item_for_impl_trait_in_trait(opaque_ty_def_id).to_def_id()
+                    tcx.associated_type_for_impl_trait_in_trait(opaque_ty_def_id).to_def_id()
                 }))
             } else {
                 &[]
@@ -216,40 +233,40 @@ fn associated_items_for_impl_trait_in_trait(tcx: TyCtxt<'_>, fn_def_id: DefId) -
             let Some(trait_fn_def_id) = tcx.associated_item(fn_def_id).trait_item_def_id else { return &[] };
 
             tcx.arena.alloc_from_iter(
-                tcx.associated_items_for_impl_trait_in_trait(trait_fn_def_id).iter().map(
-                    move |trait_assoc_def_id| {
-                        impl_associated_item_for_impl_trait_in_trait(
-                            tcx,
-                            trait_assoc_def_id.expect_local(),
-                            fn_def_id.expect_local(),
-                        )
-                        .to_def_id()
+                tcx.associated_types_for_impl_traits_in_associated_fn(trait_fn_def_id).iter().map(
+                    move |&trait_assoc_def_id| {
+                        associated_type_for_impl_trait_in_impl(tcx, trait_assoc_def_id, fn_def_id)
+                            .to_def_id()
                     },
                 ),
             )
         }
 
         def_kind => bug!(
-            "associated_items_for_impl_trait_in_trait: {:?} should be Trait or Impl but is {:?}",
+            "associated_types_for_impl_traits_in_associated_fn: {:?} should be Trait or Impl but is {:?}",
             parent_def_id,
             def_kind
         ),
     }
 }
 
-/// Given an `opaque_ty_def_id` corresponding to an impl trait in trait, create and return the
-/// corresponding associated item.
-fn associated_item_for_impl_trait_in_trait(
+/// Given an `opaque_ty_def_id` corresponding to an `impl Trait` in an associated
+/// function from a trait, synthesize an associated type for that `impl Trait`
+/// that inherits properties that we infer from the method and the opaque type.
+fn associated_type_for_impl_trait_in_trait(
     tcx: TyCtxt<'_>,
     opaque_ty_def_id: LocalDefId,
 ) -> LocalDefId {
-    let fn_def_id = tcx.impl_trait_in_trait_parent(opaque_ty_def_id.to_def_id());
-    let trait_def_id = tcx.parent(fn_def_id);
+    let (hir::OpaqueTyOrigin::FnReturn(fn_def_id) | hir::OpaqueTyOrigin::AsyncFn(fn_def_id)) =
+        tcx.hir().expect_item(opaque_ty_def_id).expect_opaque_ty().origin
+    else {
+        bug!("expected opaque for {opaque_ty_def_id:?}");
+    };
+    let trait_def_id = tcx.local_parent(fn_def_id);
     assert_eq!(tcx.def_kind(trait_def_id), DefKind::Trait);
 
     let span = tcx.def_span(opaque_ty_def_id);
-    let trait_assoc_ty =
-        tcx.at(span).create_def(trait_def_id.expect_local(), DefPathData::ImplTraitAssocTy);
+    let trait_assoc_ty = tcx.at(span).create_def(trait_def_id, DefPathData::ImplTraitAssocTy);
 
     let local_def_id = trait_assoc_ty.def_id();
     let def_id = local_def_id.to_def_id();
@@ -263,12 +280,6 @@ fn associated_item_for_impl_trait_in_trait(
     // Copy span of the opaque.
     trait_assoc_ty.def_ident_span(Some(span));
 
-    // Add the def_id of the function and opaque that generated this synthesized associated type.
-    trait_assoc_ty.opt_rpitit_info(Some(ImplTraitInTraitData::Trait {
-        fn_def_id,
-        opaque_def_id: opaque_ty_def_id.to_def_id(),
-    }));
-
     trait_assoc_ty.associated_item(ty::AssocItem {
         name: kw::Empty,
         kind: ty::AssocKind::Type,
@@ -276,6 +287,10 @@ fn associated_item_for_impl_trait_in_trait(
         trait_item_def_id: None,
         container: ty::TraitContainer,
         fn_has_self_parameter: false,
+        opt_rpitit_info: Some(ImplTraitInTraitData::Trait {
+            fn_def_id: fn_def_id.to_def_id(),
+            opaque_def_id: opaque_ty_def_id.to_def_id(),
+        }),
     });
 
     // Copy visility of the containing function.
@@ -287,40 +302,70 @@ fn associated_item_for_impl_trait_in_trait(
     // Copy type_of of the opaque.
     trait_assoc_ty.type_of(ty::EarlyBinder(tcx.mk_opaque(
         opaque_ty_def_id.to_def_id(),
-        InternalSubsts::identity_for_item(tcx, opaque_ty_def_id.to_def_id()),
+        InternalSubsts::identity_for_item(tcx, opaque_ty_def_id),
     )));
 
-    // Copy generics_of of the opaque.
-    trait_assoc_ty.generics_of(tcx.generics_of(opaque_ty_def_id).clone());
+    trait_assoc_ty.is_type_alias_impl_trait(false);
+
+    // Copy generics_of of the opaque type item but the trait is the parent.
+    trait_assoc_ty.generics_of({
+        let opaque_ty_generics = tcx.generics_of(opaque_ty_def_id);
+        let opaque_ty_parent_count = opaque_ty_generics.parent_count;
+        let mut params = opaque_ty_generics.params.clone();
+
+        let parent_generics = tcx.generics_of(trait_def_id);
+        let parent_count = parent_generics.parent_count + parent_generics.params.len();
+
+        let mut trait_fn_params = tcx.generics_of(fn_def_id).params.clone();
+
+        for param in &mut params {
+            param.index = param.index + parent_count as u32 + trait_fn_params.len() as u32
+                - opaque_ty_parent_count as u32;
+        }
+
+        trait_fn_params.extend(params);
+        params = trait_fn_params;
+
+        let param_def_id_to_index =
+            params.iter().map(|param| (param.def_id, param.index)).collect();
+
+        ty::Generics {
+            parent: Some(trait_def_id.to_def_id()),
+            parent_count,
+            params,
+            param_def_id_to_index,
+            has_self: false,
+            has_late_bound_regions: opaque_ty_generics.has_late_bound_regions,
+        }
+    });
 
     // There are no predicates for the synthesized associated type.
     trait_assoc_ty.explicit_predicates_of(ty::GenericPredicates {
-        parent: Some(trait_def_id),
+        parent: Some(trait_def_id.to_def_id()),
         predicates: &[],
     });
 
     // There are no inferred outlives for the synthesized associated type.
     trait_assoc_ty.inferred_outlives_of(&[]);
 
-    // FIXME implement this.
-    trait_assoc_ty.explicit_item_bounds(&[]);
-
     local_def_id
 }
 
-/// Given an `trait_assoc_def_id` that corresponds to a previously synthesized impl trait in trait
-/// into an associated type and an `impl_def_id` corresponding to an impl block, create and return
-/// the corresponding associated item inside the impl block.
-fn impl_associated_item_for_impl_trait_in_trait(
+/// Given an `trait_assoc_def_id` corresponding to an associated item synthesized
+/// from an `impl Trait` in an associated function from a trait, and an
+/// `impl_fn_def_id` that represents an implementation of the associated function
+/// that the `impl Trait` comes from, synthesize an associated type for that `impl Trait`
+/// that inherits properties that we infer from the method and the associated type.
+fn associated_type_for_impl_trait_in_impl(
     tcx: TyCtxt<'_>,
-    trait_assoc_def_id: LocalDefId,
+    trait_assoc_def_id: DefId,
     impl_fn_def_id: LocalDefId,
 ) -> LocalDefId {
-    let impl_def_id = tcx.local_parent(impl_fn_def_id);
+    let impl_local_def_id = tcx.local_parent(impl_fn_def_id);
 
     // FIXME fix the span, we probably want the def_id of the return type of the function
     let span = tcx.def_span(impl_fn_def_id);
-    let impl_assoc_ty = tcx.at(span).create_def(impl_def_id, DefPathData::ImplTraitAssocTy);
+    let impl_assoc_ty = tcx.at(span).create_def(impl_local_def_id, DefPathData::ImplTraitAssocTy);
 
     let local_def_id = impl_assoc_ty.def_id();
     let def_id = local_def_id.to_def_id();
@@ -331,27 +376,61 @@ fn impl_associated_item_for_impl_trait_in_trait(
     // `opt_local_def_id_to_hir_id` with `None`.
     impl_assoc_ty.opt_local_def_id_to_hir_id(None);
 
-    // Add the def_id of the function that generated this synthesized associated type.
-    impl_assoc_ty.opt_rpitit_info(Some(ImplTraitInTraitData::Impl {
-        fn_def_id: impl_fn_def_id.to_def_id(),
-    }));
+    // Copy span of the opaque.
+    impl_assoc_ty.def_ident_span(Some(span));
 
     impl_assoc_ty.associated_item(ty::AssocItem {
         name: kw::Empty,
         kind: ty::AssocKind::Type,
         def_id,
-        trait_item_def_id: Some(trait_assoc_def_id.to_def_id()),
+        trait_item_def_id: Some(trait_assoc_def_id),
         container: ty::ImplContainer,
         fn_has_self_parameter: false,
+        opt_rpitit_info: Some(ImplTraitInTraitData::Impl { fn_def_id: impl_fn_def_id.to_def_id() }),
     });
 
+    // Copy visility of the containing function.
+    impl_assoc_ty.visibility(tcx.visibility(impl_fn_def_id));
+
     // Copy impl_defaultness of the containing function.
     impl_assoc_ty.impl_defaultness(tcx.impl_defaultness(impl_fn_def_id));
 
-    // Copy generics_of the trait's associated item.
-    // FIXME: This is not correct, in particular the parent is going to be wrong. So we would need
-    // to copy from trait_assoc_def_id and adjust things.
-    impl_assoc_ty.generics_of(tcx.generics_of(trait_assoc_def_id).clone());
+    // Copy generics_of the trait's associated item but the impl as the parent.
+    // FIXME(-Zlower-impl-trait-in-trait-to-assoc-ty) resolves to the trait instead of the impl
+    // generics.
+    impl_assoc_ty.generics_of({
+        let trait_assoc_generics = tcx.generics_of(trait_assoc_def_id);
+        let trait_assoc_parent_count = trait_assoc_generics.parent_count;
+        let mut params = trait_assoc_generics.params.clone();
+
+        let parent_generics = tcx.generics_of(impl_local_def_id.to_def_id());
+        let parent_count = parent_generics.parent_count + parent_generics.params.len();
+
+        for param in &mut params {
+            param.index = param.index + parent_count as u32 - trait_assoc_parent_count as u32;
+        }
+
+        let param_def_id_to_index =
+            params.iter().map(|param| (param.def_id, param.index)).collect();
+
+        ty::Generics {
+            parent: Some(impl_local_def_id.to_def_id()),
+            parent_count,
+            params,
+            param_def_id_to_index,
+            has_self: false,
+            has_late_bound_regions: trait_assoc_generics.has_late_bound_regions,
+        }
+    });
+
+    // There are no predicates for the synthesized associated type.
+    impl_assoc_ty.explicit_predicates_of(ty::GenericPredicates {
+        parent: Some(impl_local_def_id.to_def_id()),
+        predicates: &[],
+    });
+
+    // There are no inferred outlives for the synthesized associated type.
+    impl_assoc_ty.inferred_outlives_of(&[]);
 
     local_def_id
 }
diff --git a/compiler/rustc_ty_utils/src/common_traits.rs b/compiler/rustc_ty_utils/src/common_traits.rs
index d3169b6d9..3b1abdcb2 100644
--- a/compiler/rustc_ty_utils/src/common_traits.rs
+++ b/compiler/rustc_ty_utils/src/common_traits.rs
@@ -3,7 +3,6 @@
 use rustc_hir::lang_items::LangItem;
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_span::DUMMY_SP;
 use rustc_trait_selection::traits;
 
 fn is_copy_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
@@ -30,7 +29,7 @@ fn is_item_raw<'tcx>(
     let (param_env, ty) = query.into_parts();
     let trait_def_id = tcx.require_lang_item(item, None);
     let infcx = tcx.infer_ctxt().build();
-    traits::type_known_to_meet_bound_modulo_regions(&infcx, param_env, ty, trait_def_id, DUMMY_SP)
+    traits::type_known_to_meet_bound_modulo_regions(&infcx, param_env, ty, trait_def_id)
 }
 
 pub(crate) fn provide(providers: &mut ty::query::Providers) {
diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs
index f26352716..b67607a4d 100644
--- a/compiler/rustc_ty_utils/src/consts.rs
+++ b/compiler/rustc_ty_utils/src/consts.rs
@@ -8,7 +8,7 @@ use rustc_middle::ty::abstract_const::CastKind;
 use rustc_middle::ty::{self, Expr, TyCtxt, TypeVisitableExt};
 use rustc_middle::{mir, thir};
 use rustc_span::Span;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{VariantIdx, FIRST_VARIANT};
 
 use std::iter;
 
@@ -44,7 +44,7 @@ pub(crate) fn destructure_const<'tcx>(
                 let (head, rest) = branches.split_first().unwrap();
                 (VariantIdx::from_u32(head.unwrap_leaf().try_to_u32().unwrap()), rest)
             } else {
-                (VariantIdx::from_u32(0), branches)
+                (FIRST_VARIANT, branches)
             };
             let fields = &def.variant(variant_idx).fields;
             let mut field_consts = Vec::with_capacity(fields.len());
@@ -425,7 +425,6 @@ pub fn provide(providers: &mut ty::query::Providers) {
     *providers = ty::query::Providers {
         destructure_const,
         thir_abstract_const: |tcx, def_id| {
-            let def_id = def_id.expect_local();
             if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) {
                 tcx.thir_abstract_const_of_const_arg(def)
             } else {
diff --git a/compiler/rustc_ty_utils/src/errors.rs b/compiler/rustc_ty_utils/src/errors.rs
index ab3e62f04..3d3fc50e6 100644
--- a/compiler/rustc_ty_utils/src/errors.rs
+++ b/compiler/rustc_ty_utils/src/errors.rs
@@ -67,3 +67,36 @@ pub enum GenericConstantTooComplexSub {
     #[label(ty_utils_operation_not_supported)]
     OperationNotSupported(#[primary_span] Span),
 }
+
+#[derive(Diagnostic)]
+#[diag(ty_utils_unexpected_fnptr_associated_item)]
+pub struct UnexpectedFnPtrAssociatedItem {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ty_utils_zero_length_simd_type)]
+pub struct ZeroLengthSimdType<'tcx> {
+    pub ty: Ty<'tcx>,
+}
+
+#[derive(Diagnostic)]
+#[diag(ty_utils_multiple_array_fields_simd_type)]
+pub struct MultipleArrayFieldsSimdType<'tcx> {
+    pub ty: Ty<'tcx>,
+}
+
+#[derive(Diagnostic)]
+#[diag(ty_utils_oversized_simd_type)]
+pub struct OversizedSimdType<'tcx> {
+    pub ty: Ty<'tcx>,
+    pub max_lanes: u64,
+}
+
+#[derive(Diagnostic)]
+#[diag(ty_utils_non_primitive_simd_type)]
+pub struct NonPrimitiveSimdType<'tcx> {
+    pub ty: Ty<'tcx>,
+    pub e_ty: Ty<'tcx>,
+}
diff --git a/compiler/rustc_ty_utils/src/implied_bounds.rs b/compiler/rustc_ty_utils/src/implied_bounds.rs
index 7fecee2a3..56d6cc28b 100644
--- a/compiler/rustc_ty_utils/src/implied_bounds.rs
+++ b/compiler/rustc_ty_utils/src/implied_bounds.rs
@@ -1,4 +1,3 @@
-use crate::rustc_middle::ty::DefIdTree;
 use rustc_hir::{def::DefKind, def_id::DefId};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index 2eaeca73d..0a6c11809 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -8,6 +8,8 @@ use rustc_span::sym;
 use rustc_trait_selection::traits;
 use traits::{translate_substs, Reveal};
 
+use crate::errors::UnexpectedFnPtrAssociatedItem;
+
 fn resolve_instance<'tcx>(
     tcx: TyCtxt<'tcx>,
     key: ty::ParamEnvAnd<'tcx, (DefId, SubstsRef<'tcx>)>,
@@ -243,7 +245,8 @@ fn resolve_associated_item<'tcx>(
             }
         }
         traits::ImplSource::Builtin(..) => {
-            if Some(trait_ref.def_id) == tcx.lang_items().clone_trait() {
+            let lang_items = tcx.lang_items();
+            if Some(trait_ref.def_id) == lang_items.clone_trait() {
                 // FIXME(eddyb) use lang items for methods instead of names.
                 let name = tcx.item_name(trait_item_id);
                 if name == sym::clone {
@@ -270,6 +273,21 @@ fn resolve_associated_item<'tcx>(
                     let substs = tcx.erase_regions(rcvr_substs);
                     Some(ty::Instance::new(trait_item_id, substs))
                 }
+            } else if Some(trait_ref.def_id) == lang_items.fn_ptr_trait() {
+                if lang_items.fn_ptr_addr() == Some(trait_item_id) {
+                    let self_ty = trait_ref.self_ty();
+                    if !matches!(self_ty.kind(), ty::FnPtr(..)) {
+                        return Ok(None);
+                    }
+                    Some(Instance {
+                        def: ty::InstanceDef::FnPtrAddrShim(trait_item_id, self_ty),
+                        substs: rcvr_substs,
+                    })
+                } else {
+                    tcx.sess.emit_fatal(UnexpectedFnPtrAssociatedItem {
+                        span: tcx.def_span(trait_item_id),
+                    })
+                }
             } else {
                 None
             }
diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs
index e3132fcc4..63ef1c724 100644
--- a/compiler/rustc_ty_utils/src/layout.rs
+++ b/compiler/rustc_ty_utils/src/layout.rs
@@ -1,7 +1,7 @@
 use hir::def_id::DefId;
 use rustc_hir as hir;
 use rustc_index::bit_set::BitSet;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::{IndexSlice, IndexVec};
 use rustc_middle::mir::{GeneratorLayout, GeneratorSavedLocal};
 use rustc_middle::ty::layout::{
     IntegerExt, LayoutCx, LayoutError, LayoutOf, TyAndLayout, MAX_SIMD_LANES,
@@ -17,6 +17,9 @@ use rustc_target::abi::*;
 use std::fmt::Debug;
 use std::iter;
 
+use crate::errors::{
+    MultipleArrayFieldsSimdType, NonPrimitiveSimdType, OversizedSimdType, ZeroLengthSimdType,
+};
 use crate::layout_sanity_check::sanity_check_layout;
 
 pub fn provide(providers: &mut ty::query::Providers) {
@@ -62,23 +65,10 @@ fn layout_of<'tcx>(
     Ok(layout)
 }
 
-// Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`.
-// This is used to go between `memory_index` (source field order to memory order)
-// and `inverse_memory_index` (memory order to source field order).
-// See also `FieldsShape::Arbitrary::memory_index` for more details.
-// FIXME(eddyb) build a better abstraction for permutations, if possible.
-fn invert_mapping(map: &[u32]) -> Vec<u32> {
-    let mut inverse = vec![0; map.len()];
-    for i in 0..map.len() {
-        inverse[map[i] as usize] = i as u32;
-    }
-    inverse
-}
-
 fn univariant_uninterned<'tcx>(
     cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
     ty: Ty<'tcx>,
-    fields: &[Layout<'_>],
+    fields: &IndexSlice<FieldIdx, Layout<'_>>,
     repr: &ReprOptions,
     kind: StructKind,
 ) -> Result<LayoutS, LayoutError<'tcx>> {
@@ -106,7 +96,7 @@ fn layout_of_uncached<'tcx>(
     };
     let scalar = |value: Primitive| tcx.mk_layout(LayoutS::scalar(cx, scalar_unit(value)));
 
-    let univariant = |fields: &[Layout<'_>], repr: &ReprOptions, kind| {
+    let univariant = |fields: &IndexSlice<FieldIdx, Layout<'_>>, repr: &ReprOptions, kind| {
         Ok(tcx.mk_layout(univariant_uninterned(cx, ty, fields, repr, kind)?))
     };
     debug_assert!(!ty.has_non_region_infer());
@@ -156,7 +146,11 @@ fn layout_of_uncached<'tcx>(
 
             let unsized_part = tcx.struct_tail_erasing_lifetimes(pointee, param_env);
 
-            let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type() {
+            let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type()
+                // Projection eagerly bails out when the pointee references errors,
+                // fall back to structurally deducing metadata.
+                && !pointee.references_error()
+            {
                 let metadata_ty = tcx.normalize_erasing_regions(
                     param_env,
                     tcx.mk_projection(metadata_def_id, [pointee]),
@@ -223,7 +217,7 @@ fn layout_of_uncached<'tcx>(
             let largest_niche = if count != 0 { element.largest_niche } else { None };
 
             tcx.mk_layout(LayoutS {
-                variants: Variants::Single { index: VariantIdx::new(0) },
+                variants: Variants::Single { index: FIRST_VARIANT },
                 fields: FieldsShape::Array { stride: element.size, count },
                 abi,
                 largest_niche,
@@ -234,7 +228,7 @@ fn layout_of_uncached<'tcx>(
         ty::Slice(element) => {
             let element = cx.layout_of(element)?;
             tcx.mk_layout(LayoutS {
-                variants: Variants::Single { index: VariantIdx::new(0) },
+                variants: Variants::Single { index: FIRST_VARIANT },
                 fields: FieldsShape::Array { stride: element.size, count: 0 },
                 abi: Abi::Aggregate { sized: false },
                 largest_niche: None,
@@ -243,7 +237,7 @@ fn layout_of_uncached<'tcx>(
             })
         }
         ty::Str => tcx.mk_layout(LayoutS {
-            variants: Variants::Single { index: VariantIdx::new(0) },
+            variants: Variants::Single { index: FIRST_VARIANT },
             fields: FieldsShape::Array { stride: Size::from_bytes(1), count: 0 },
             abi: Abi::Aggregate { sized: false },
             largest_niche: None,
@@ -252,12 +246,14 @@ fn layout_of_uncached<'tcx>(
         }),
 
         // Odd unit types.
-        ty::FnDef(..) => univariant(&[], &ReprOptions::default(), StructKind::AlwaysSized)?,
+        ty::FnDef(..) => {
+            univariant(IndexSlice::empty(), &ReprOptions::default(), StructKind::AlwaysSized)?
+        }
         ty::Dynamic(_, _, ty::Dyn) | ty::Foreign(..) => {
             let mut unit = univariant_uninterned(
                 cx,
                 ty,
-                &[],
+                IndexSlice::empty(),
                 &ReprOptions::default(),
                 StructKind::AlwaysSized,
             )?;
@@ -273,7 +269,7 @@ fn layout_of_uncached<'tcx>(
         ty::Closure(_, ref substs) => {
             let tys = substs.as_closure().upvar_tys();
             univariant(
-                &tys.map(|ty| Ok(cx.layout_of(ty)?.layout)).collect::<Result<Vec<_>, _>>()?,
+                &tys.map(|ty| Ok(cx.layout_of(ty)?.layout)).try_collect::<IndexVec<_, _>>()?,
                 &ReprOptions::default(),
                 StructKind::AlwaysSized,
             )?
@@ -284,7 +280,7 @@ fn layout_of_uncached<'tcx>(
                 if tys.len() == 0 { StructKind::AlwaysSized } else { StructKind::MaybeUnsized };
 
             univariant(
-                &tys.iter().map(|k| Ok(cx.layout_of(k)?.layout)).collect::<Result<Vec<_>, _>>()?,
+                &tys.iter().map(|k| Ok(cx.layout_of(k)?.layout)).try_collect::<IndexVec<_, _>>()?,
                 &ReprOptions::default(),
                 kind,
             )?
@@ -301,6 +297,8 @@ fn layout_of_uncached<'tcx>(
                 return Err(LayoutError::Unknown(ty));
             }
 
+            let fields = &def.non_enum_variant().fields;
+
             // Supported SIMD vectors are homogeneous ADTs with at least one field:
             //
             // * #[repr(simd)] struct S(T, T, T, T);
@@ -311,18 +309,22 @@ fn layout_of_uncached<'tcx>(
 
             // SIMD vectors with zero fields are not supported.
             // (should be caught by typeck)
-            if def.non_enum_variant().fields.is_empty() {
-                tcx.sess.fatal(&format!("monomorphising SIMD type `{}` of zero length", ty));
+            if fields.is_empty() {
+                tcx.sess.emit_fatal(ZeroLengthSimdType { ty })
             }
 
             // Type of the first ADT field:
-            let f0_ty = def.non_enum_variant().fields[0].ty(tcx, substs);
+            let f0_ty = fields[FieldIdx::from_u32(0)].ty(tcx, substs);
 
             // Heterogeneous SIMD vectors are not supported:
             // (should be caught by typeck)
-            for fi in &def.non_enum_variant().fields {
+            for fi in fields {
                 if fi.ty(tcx, substs) != f0_ty {
-                    tcx.sess.fatal(&format!("monomorphising heterogeneous SIMD type `{}`", ty));
+                    tcx.sess.delay_span_bug(
+                        DUMMY_SP,
+                        "#[repr(simd)] was applied to an ADT with heterogeneous field type",
+                    );
+                    return Err(LayoutError::Unknown(ty));
                 }
             }
 
@@ -337,12 +339,9 @@ fn layout_of_uncached<'tcx>(
                 // First ADT field is an array:
 
                 // SIMD vectors with multiple array fields are not supported:
-                // (should be caught by typeck)
+                // Can't be caught by typeck with a generic simd type.
                 if def.non_enum_variant().fields.len() != 1 {
-                    tcx.sess.fatal(&format!(
-                        "monomorphising SIMD type `{}` with more than one array field",
-                        ty
-                    ));
+                    tcx.sess.emit_fatal(MultipleArrayFieldsSimdType { ty });
                 }
 
                 // Extract the number of elements from the layout of the array field:
@@ -362,12 +361,9 @@ fn layout_of_uncached<'tcx>(
             //
             // Can't be caught in typeck if the array length is generic.
             if e_len == 0 {
-                tcx.sess.fatal(&format!("monomorphising SIMD type `{}` of zero length", ty));
+                tcx.sess.emit_fatal(ZeroLengthSimdType { ty });
             } else if e_len > MAX_SIMD_LANES {
-                tcx.sess.fatal(&format!(
-                    "monomorphising SIMD type `{}` of length greater than {}",
-                    ty, MAX_SIMD_LANES,
-                ));
+                tcx.sess.emit_fatal(OversizedSimdType { ty, max_lanes: MAX_SIMD_LANES });
             }
 
             // Compute the ABI of the element type:
@@ -375,11 +371,7 @@ fn layout_of_uncached<'tcx>(
             let Abi::Scalar(e_abi) = e_ly.abi else {
                 // This error isn't caught in typeck, e.g., if
                 // the element type of the vector is generic.
-                tcx.sess.fatal(&format!(
-                    "monomorphising SIMD type `{}` with a non-primitive-scalar \
-                    (integer/float/pointer) element type `{}`",
-                    ty, e_ty
-                ))
+                tcx.sess.emit_fatal(NonPrimitiveSimdType { ty, e_ty });
             };
 
             // Compute the size and alignment of the vector:
@@ -389,13 +381,13 @@ fn layout_of_uncached<'tcx>(
 
             // Compute the placement of the vector fields:
             let fields = if is_array {
-                FieldsShape::Arbitrary { offsets: vec![Size::ZERO], memory_index: vec![0] }
+                FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), memory_index: [0].into() }
             } else {
                 FieldsShape::Array { stride: e_ly.size, count: e_len }
             };
 
             tcx.mk_layout(LayoutS {
-                variants: Variants::Single { index: VariantIdx::new(0) },
+                variants: Variants::Single { index: FIRST_VARIANT },
                 fields,
                 abi: Abi::Vector { element: e_abi, count: e_len },
                 largest_niche: e_ly.largest_niche,
@@ -414,9 +406,9 @@ fn layout_of_uncached<'tcx>(
                     v.fields
                         .iter()
                         .map(|field| Ok(cx.layout_of(field.ty(tcx, substs))?.layout))
-                        .collect::<Result<Vec<_>, _>>()
+                        .try_collect::<IndexVec<_, _>>()
                 })
-                .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
+                .try_collect::<IndexVec<VariantIdx, _>>()?;
 
             if def.is_union() {
                 if def.repr().pack.is_some() && def.repr().align.is_some() {
@@ -452,7 +444,8 @@ fn layout_of_uncached<'tcx>(
                     {
                         let param_env = tcx.param_env(def.did());
                         def.is_struct()
-                            && match def.variants().iter().next().and_then(|x| x.fields.last()) {
+                            && match def.variants().iter().next().and_then(|x| x.fields.raw.last())
+                            {
                                 Some(last_field) => tcx
                                     .type_of(last_field.did)
                                     .subst_identity()
@@ -487,8 +480,7 @@ fn layout_of_uncached<'tcx>(
 enum SavedLocalEligibility {
     Unassigned,
     Assigned(VariantIdx),
-    // FIXME: Use newtype_index so we aren't wasting bytes
-    Ineligible(Option<u32>),
+    Ineligible(Option<FieldIdx>),
 }
 
 // When laying out generators, we divide our saved local fields into two
@@ -517,7 +509,7 @@ fn generator_saved_local_eligibility(
     use SavedLocalEligibility::*;
 
     let mut assignments: IndexVec<GeneratorSavedLocal, SavedLocalEligibility> =
-        IndexVec::from_elem_n(Unassigned, info.field_tys.len());
+        IndexVec::from_elem(Unassigned, &info.field_tys);
 
     // The saved locals not eligible for overlap. These will get
     // "promoted" to the prefix of our generator.
@@ -600,7 +592,7 @@ fn generator_saved_local_eligibility(
     // Write down the order of our locals that will be promoted to the prefix.
     {
         for (idx, local) in ineligible_locals.iter().enumerate() {
-            assignments[local] = Ineligible(Some(idx as u32));
+            assignments[local] = Ineligible(Some(FieldIdx::from_usize(idx)));
         }
     }
     debug!("generator saved local assignments: {:?}", assignments);
@@ -649,7 +641,7 @@ fn generator_layout<'tcx>(
         .map(|ty| Ok(cx.layout_of(ty)?.layout))
         .chain(iter::once(Ok(tag_layout)))
         .chain(promoted_layouts)
-        .collect::<Result<Vec<_>, _>>()?;
+        .try_collect::<IndexVec<_, _>>()?;
     let prefix = univariant_uninterned(
         cx,
         ty,
@@ -667,26 +659,28 @@ fn generator_layout<'tcx>(
     debug!("prefix = {:#?}", prefix);
     let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields {
         FieldsShape::Arbitrary { mut offsets, memory_index } => {
-            let mut inverse_memory_index = invert_mapping(&memory_index);
+            let mut inverse_memory_index = memory_index.invert_bijective_mapping();
 
             // "a" (`0..b_start`) and "b" (`b_start..`) correspond to
             // "outer" and "promoted" fields respectively.
-            let b_start = (tag_index + 1) as u32;
-            let offsets_b = offsets.split_off(b_start as usize);
+            let b_start = FieldIdx::from_usize(tag_index + 1);
+            let offsets_b = IndexVec::from_raw(offsets.raw.split_off(b_start.as_usize()));
             let offsets_a = offsets;
 
             // Disentangle the "a" and "b" components of `inverse_memory_index`
             // by preserving the order but keeping only one disjoint "half" each.
             // FIXME(eddyb) build a better abstraction for permutations, if possible.
-            let inverse_memory_index_b: Vec<_> =
-                inverse_memory_index.iter().filter_map(|&i| i.checked_sub(b_start)).collect();
-            inverse_memory_index.retain(|&i| i < b_start);
+            let inverse_memory_index_b: IndexVec<u32, FieldIdx> = inverse_memory_index
+                .iter()
+                .filter_map(|&i| i.as_u32().checked_sub(b_start.as_u32()).map(FieldIdx::from_u32))
+                .collect();
+            inverse_memory_index.raw.retain(|&i| i < b_start);
             let inverse_memory_index_a = inverse_memory_index;
 
             // Since `inverse_memory_index_{a,b}` each only refer to their
             // respective fields, they can be safely inverted
-            let memory_index_a = invert_mapping(&inverse_memory_index_a);
-            let memory_index_b = invert_mapping(&inverse_memory_index_b);
+            let memory_index_a = inverse_memory_index_a.invert_bijective_mapping();
+            let memory_index_b = inverse_memory_index_b.invert_bijective_mapping();
 
             let outer_fields =
                 FieldsShape::Arbitrary { offsets: offsets_a, memory_index: memory_index_a };
@@ -717,7 +711,7 @@ fn generator_layout<'tcx>(
                 ty,
                 &variant_only_tys
                     .map(|ty| Ok(cx.layout_of(ty)?.layout))
-                    .collect::<Result<Vec<_>, _>>()?,
+                    .try_collect::<IndexVec<_, _>>()?,
                 &ReprOptions::default(),
                 StructKind::Prefixed(prefix_size, prefix_align.abi),
             )?;
@@ -736,13 +730,16 @@ fn generator_layout<'tcx>(
             // promoted fields were being used, but leave the elements not in the
             // subset as `INVALID_FIELD_IDX`, which we can filter out later to
             // obtain a valid (bijective) mapping.
-            const INVALID_FIELD_IDX: u32 = !0;
-            let mut combined_inverse_memory_index =
-                vec![INVALID_FIELD_IDX; promoted_memory_index.len() + memory_index.len()];
+            const INVALID_FIELD_IDX: FieldIdx = FieldIdx::MAX;
+            debug_assert!(variant_fields.next_index() <= INVALID_FIELD_IDX);
+
+            let mut combined_inverse_memory_index = IndexVec::from_elem_n(
+                INVALID_FIELD_IDX,
+                promoted_memory_index.len() + memory_index.len(),
+            );
             let mut offsets_and_memory_index = iter::zip(offsets, memory_index);
             let combined_offsets = variant_fields
-                .iter()
-                .enumerate()
+                .iter_enumerated()
                 .map(|(i, local)| {
                     let (offset, memory_index) = match assignments[*local] {
                         Unassigned => bug!(),
@@ -751,19 +748,19 @@ fn generator_layout<'tcx>(
                             (offset, promoted_memory_index.len() as u32 + memory_index)
                         }
                         Ineligible(field_idx) => {
-                            let field_idx = field_idx.unwrap() as usize;
+                            let field_idx = field_idx.unwrap();
                             (promoted_offsets[field_idx], promoted_memory_index[field_idx])
                         }
                     };
-                    combined_inverse_memory_index[memory_index as usize] = i as u32;
+                    combined_inverse_memory_index[memory_index] = i;
                     offset
                 })
                 .collect();
 
             // Remove the unused slots and invert the mapping to obtain the
             // combined `memory_index` (also see previous comment).
-            combined_inverse_memory_index.retain(|&i| i != INVALID_FIELD_IDX);
-            let combined_memory_index = invert_mapping(&combined_inverse_memory_index);
+            combined_inverse_memory_index.raw.retain(|&i| i != INVALID_FIELD_IDX);
+            let combined_memory_index = combined_inverse_memory_index.invert_bijective_mapping();
 
             variant.fields = FieldsShape::Arbitrary {
                 offsets: combined_offsets,
@@ -774,7 +771,7 @@ fn generator_layout<'tcx>(
             align = align.max(variant.align);
             Ok(variant)
         })
-        .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
+        .try_collect::<IndexVec<VariantIdx, _>>()?;
 
     size = size.align_to(align.abi);
 
diff --git a/compiler/rustc_ty_utils/src/lib.rs b/compiler/rustc_ty_utils/src/lib.rs
index 35f468aa9..2613445f3 100644
--- a/compiler/rustc_ty_utils/src/lib.rs
+++ b/compiler/rustc_ty_utils/src/lib.rs
@@ -5,10 +5,13 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(iterator_try_collect)]
 #![feature(let_chains)]
 #![feature(never_type)]
 #![feature(box_patterns)]
 #![recursion_limit = "256"]
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 
 #[macro_use]
 extern crate rustc_middle;
@@ -33,7 +36,7 @@ pub mod representability;
 mod structural_match;
 mod ty;
 
-fluent_messages! { "../locales/en-US.ftl" }
+fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     abi::provide(providers);
diff --git a/compiler/rustc_ty_utils/src/representability.rs b/compiler/rustc_ty_utils/src/representability.rs
index 591017eec..26d6deab8 100644
--- a/compiler/rustc_ty_utils/src/representability.rs
+++ b/compiler/rustc_ty_utils/src/representability.rs
@@ -4,7 +4,7 @@ use rustc_hir::def::DefKind;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, Representability, Ty, TyCtxt};
-use rustc_span::def_id::{DefId, LocalDefId};
+use rustc_span::def_id::LocalDefId;
 
 pub fn provide(providers: &mut Providers) {
     *providers =
@@ -85,7 +85,7 @@ fn representability_adt_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Representab
     Representability::Representable
 }
 
-fn params_in_repr(tcx: TyCtxt<'_>, def_id: DefId) -> BitSet<u32> {
+fn params_in_repr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> BitSet<u32> {
     let adt_def = tcx.adt_def(def_id);
     let generics = tcx.generics_of(def_id);
     let mut params_in_repr = BitSet::new_empty(generics.params.len());
diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs
index 18159778a..cb06c7acf 100644
--- a/compiler/rustc_ty_utils/src/ty.rs
+++ b/compiler/rustc_ty_utils/src/ty.rs
@@ -3,11 +3,12 @@ use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::ty::{
-    self, Binder, EarlyBinder, Predicate, PredicateKind, ToPredicate, Ty, TyCtxt,
-    TypeSuperVisitable, TypeVisitable, TypeVisitor,
+    self, Binder, EarlyBinder, ImplTraitInTraitData, Predicate, PredicateKind, ToPredicate, Ty,
+    TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor,
 };
 use rustc_session::config::TraitSolver;
-use rustc_span::def_id::{DefId, CRATE_DEF_ID};
+use rustc_span::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
+use rustc_span::DUMMY_SP;
 use rustc_trait_selection::traits;
 
 fn sized_constraint_for_ty<'tcx>(
@@ -76,8 +77,8 @@ fn sized_constraint_for_ty<'tcx>(
     result
 }
 
-fn impl_defaultness(tcx: TyCtxt<'_>, def_id: DefId) -> hir::Defaultness {
-    match tcx.hir().get_by_def_id(def_id.expect_local()) {
+fn impl_defaultness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Defaultness {
+    match tcx.hir().get_by_def_id(def_id) {
         hir::Node::Item(hir::Item { kind: hir::ItemKind::Impl(impl_), .. }) => impl_.defaultness,
         hir::Node::ImplItem(hir::ImplItem { defaultness, .. })
         | hir::Node::TraitItem(hir::TraitItem { defaultness, .. }) => *defaultness,
@@ -106,7 +107,7 @@ fn adt_sized_constraint(tcx: TyCtxt<'_>, def_id: DefId) -> &[Ty<'_>] {
     let result = tcx.mk_type_list_from_iter(
         def.variants()
             .iter()
-            .flat_map(|v| v.fields.last())
+            .filter_map(|v| v.fields.raw.last())
             .flat_map(|f| sized_constraint_for_ty(tcx, def, tcx.type_of(f.did).subst_identity())),
     );
 
@@ -121,6 +122,20 @@ fn param_env(tcx: TyCtxt<'_>, def_id: DefId) -> ty::ParamEnv<'_> {
     let ty::InstantiatedPredicates { mut predicates, .. } =
         tcx.predicates_of(def_id).instantiate_identity(tcx);
 
+    // When computing the param_env of an RPITIT, use predicates of the containing function,
+    // *except* for the additional assumption that the RPITIT normalizes to the trait method's
+    // default opaque type. This is needed to properly check the item bounds of the assoc
+    // type hold (`check_type_bounds`), since that method already installs a similar projection
+    // bound, so they will conflict.
+    // FIXME(-Zlower-impl-trait-in-trait-to-assoc-ty): I don't like this, we should
+    // at least be making sure that the generics in RPITITs and their parent fn don't
+    // get out of alignment, or else we do actually need to substitute these predicates.
+    if let Some(ImplTraitInTraitData::Trait { fn_def_id, .. })
+    | Some(ImplTraitInTraitData::Impl { fn_def_id, .. }) = tcx.opt_rpitit_info(def_id)
+    {
+        predicates = tcx.predicates_of(fn_def_id).instantiate_identity(tcx).predicates;
+    }
+
     // Finally, we have to normalize the bounds in the environment, in
     // case they contain any associated type projections. This process
     // can yield errors if the put in illegal associated types, like
@@ -142,17 +157,21 @@ fn param_env(tcx: TyCtxt<'_>, def_id: DefId) -> ty::ParamEnv<'_> {
         && tcx.associated_item(def_id).container == ty::AssocItemContainer::TraitContainer
     {
         let sig = tcx.fn_sig(def_id).subst_identity();
-        sig.visit_with(&mut ImplTraitInTraitFinder {
+        // We accounted for the binder of the fn sig, so skip the binder.
+        sig.skip_binder().visit_with(&mut ImplTraitInTraitFinder {
             tcx,
             fn_def_id: def_id,
             bound_vars: sig.bound_vars(),
             predicates: &mut predicates,
             seen: FxHashSet::default(),
+            depth: ty::INNERMOST,
         });
     }
 
     let local_did = def_id.as_local();
-    let hir_id = local_did.map(|def_id| tcx.hir().local_def_id_to_hir_id(def_id));
+    // FIXME(-Zlower-impl-trait-in-trait-to-assoc-ty): This isn't correct for
+    // RPITITs in const trait fn.
+    let hir_id = local_did.and_then(|def_id| tcx.opt_local_def_id_to_hir_id(def_id));
 
     // FIXME(consts): This is not exactly in line with the constness query.
     let constness = match hir_id {
@@ -244,27 +263,68 @@ struct ImplTraitInTraitFinder<'a, 'tcx> {
     fn_def_id: DefId,
     bound_vars: &'tcx ty::List<ty::BoundVariableKind>,
     seen: FxHashSet<DefId>,
+    depth: ty::DebruijnIndex,
 }
 
 impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ImplTraitInTraitFinder<'_, 'tcx> {
+    fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
+        &mut self,
+        binder: &ty::Binder<'tcx, T>,
+    ) -> std::ops::ControlFlow<Self::BreakTy> {
+        self.depth.shift_in(1);
+        let binder = binder.super_visit_with(self);
+        self.depth.shift_out(1);
+        binder
+    }
+
     fn visit_ty(&mut self, ty: Ty<'tcx>) -> std::ops::ControlFlow<Self::BreakTy> {
-        if let ty::Alias(ty::Projection, alias_ty) = *ty.kind()
-            && self.tcx.def_kind(alias_ty.def_id) == DefKind::ImplTraitPlaceholder
-            && self.tcx.impl_trait_in_trait_parent(alias_ty.def_id) == self.fn_def_id
-            && self.seen.insert(alias_ty.def_id)
+        if let ty::Alias(ty::Projection, unshifted_alias_ty) = *ty.kind()
+            && self.tcx.is_impl_trait_in_trait(unshifted_alias_ty.def_id)
+            && self.tcx.impl_trait_in_trait_parent_fn(unshifted_alias_ty.def_id) == self.fn_def_id
+            && self.seen.insert(unshifted_alias_ty.def_id)
         {
+            // We have entered some binders as we've walked into the
+            // bounds of the RPITIT. Shift these binders back out when
+            // constructing the top-level projection predicate.
+            let shifted_alias_ty = self.tcx.fold_regions(unshifted_alias_ty, |re, depth| {
+                if let ty::ReLateBound(index, bv) = re.kind() {
+                    if depth != ty::INNERMOST {
+                        return self.tcx.mk_re_error_with_message(
+                            DUMMY_SP,
+                            "we shouldn't walk non-predicate binders with `impl Trait`...",
+                        );
+                    }
+                    self.tcx.mk_re_late_bound(index.shifted_out_to_binder(self.depth), bv)
+                } else {
+                    re
+                }
+            });
+
+            // If we're lowering to associated item, install the opaque type which is just
+            // the `type_of` of the trait's associated item. If we're using the old lowering
+            // strategy, then just reinterpret the associated type like an opaque :^)
+            let default_ty = if self.tcx.lower_impl_trait_in_trait_to_assoc_ty() {
+                self.tcx.type_of(shifted_alias_ty.def_id).subst(self.tcx, shifted_alias_ty.substs)
+            } else {
+                self.tcx.mk_alias(ty::Opaque, shifted_alias_ty)
+            };
+
             self.predicates.push(
                 ty::Binder::bind_with_vars(
-                    ty::ProjectionPredicate {
-                        projection_ty: alias_ty,
-                        term: self.tcx.mk_alias(ty::Opaque, alias_ty).into(),
-                    },
+                    ty::ProjectionPredicate { projection_ty: shifted_alias_ty, term: default_ty.into() },
                     self.bound_vars,
                 )
                 .to_predicate(self.tcx),
             );
 
-            for bound in self.tcx.item_bounds(alias_ty.def_id).subst_iter(self.tcx, alias_ty.substs)
+            // We walk the *un-shifted* alias ty, because we're tracking the de bruijn
+            // binder depth, and if we were to walk `shifted_alias_ty` instead, we'd
+            // have to reset `self.depth` back to `ty::INNERMOST` or something. It's
+            // easier to just do this.
+            for bound in self
+                .tcx
+                .item_bounds(unshifted_alias_ty.def_id)
+                .subst_iter(self.tcx, unshifted_alias_ty.substs)
             {
                 bound.visit_with(self);
             }
@@ -456,8 +516,8 @@ fn issue33140_self_ty(tcx: TyCtxt<'_>, def_id: DefId) -> Option<EarlyBinder<Ty<'
 }
 
 /// Check if a function is async.
-fn asyncness(tcx: TyCtxt<'_>, def_id: DefId) -> hir::IsAsync {
-    let node = tcx.hir().get_by_def_id(def_id.expect_local());
+fn asyncness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::IsAsync {
+    let node = tcx.hir().get_by_def_id(def_id);
     node.fn_sig().map_or(hir::IsAsync::NotAsync, |sig| sig.header.asyncness)
 }
 
@@ -482,7 +542,7 @@ fn unsizing_params_for_adt<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> BitSet<u32
 
     // The last field of the structure has to exist and contain type/const parameters.
     let Some((tail_field, prefix_fields)) =
-        def.non_enum_variant().fields.split_last() else
+        def.non_enum_variant().fields.raw.split_last() else
     {
         return BitSet::new_empty(num_params);
     };
diff --git a/compiler/rustc_type_ir/src/fold.rs b/compiler/rustc_type_ir/src/fold.rs
index ee4ef57c3..3a053d4c6 100644
--- a/compiler/rustc_type_ir/src/fold.rs
+++ b/compiler/rustc_type_ir/src/fold.rs
@@ -18,7 +18,7 @@
 //!     It defines a "skeleton" of how they should be folded.
 //! - `TypeSuperFoldable`. This is implemented only for each type of interest,
 //!   and defines the folding "skeleton" for these types.
-//! - `TypeFolder`/`FallibleTypeFolder. One of these is implemented for each
+//! - `TypeFolder`/`FallibleTypeFolder`. One of these is implemented for each
 //!   folder. This defines how types of interest are folded.
 //!
 //! This means each fold is a mixture of (a) generic folding operations, and (b)
diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs
index 5a991e03d..a3c98ae00 100644
--- a/compiler/rustc_type_ir/src/lib.rs
+++ b/compiler/rustc_type_ir/src/lib.rs
@@ -83,7 +83,7 @@ pub trait CollectAndApply<T, R>: Sized {
     /// Produce a result of type `Self::Output` from `iter`. The result will
     /// typically be produced by applying `f` on the elements produced by
     /// `iter`, though this may not happen in some impls, e.g. if an error
-    /// occured during iteration.
+    /// occurred during iteration.
     fn collect_and_apply<I, F>(iter: I, f: F) -> Self::Output
     where
         I: Iterator<Item = Self>,
@@ -432,6 +432,17 @@ impl IntTy {
             _ => *self,
         }
     }
+
+    pub fn to_unsigned(self) -> UintTy {
+        match self {
+            IntTy::Isize => UintTy::Usize,
+            IntTy::I8 => UintTy::U8,
+            IntTy::I16 => UintTy::U16,
+            IntTy::I32 => UintTy::U32,
+            IntTy::I64 => UintTy::U64,
+            IntTy::I128 => UintTy::U128,
+        }
+    }
 }
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy, Debug)]
@@ -479,6 +490,17 @@ impl UintTy {
             _ => *self,
         }
     }
+
+    pub fn to_signed(self) -> IntTy {
+        match self {
+            UintTy::Usize => IntTy::Isize,
+            UintTy::U8 => IntTy::I8,
+            UintTy::U16 => IntTy::I16,
+            UintTy::U32 => IntTy::I32,
+            UintTy::U64 => IntTy::I64,
+            UintTy::U128 => IntTy::I128,
+        }
+    }
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
diff --git a/compiler/rustc_type_ir/src/sty.rs b/compiler/rustc_type_ir/src/sty.rs
index ebe2b76ae..62e699eef 100644
--- a/compiler/rustc_type_ir/src/sty.rs
+++ b/compiler/rustc_type_ir/src/sty.rs
@@ -167,7 +167,7 @@ pub enum TyKind<I: Interner> {
     /// lifetimes bound by the witness itself.
     ///
     /// This variant is only using when `drop_tracking_mir` is set.
-    /// This contains the `DefId` and the `SubstRef` of the generator.
+    /// This contains the `DefId` and the `SubstsRef` of the generator.
     /// The actual witness types are computed on MIR by the `mir_generator_witnesses` query.
     ///
     /// Looking at the following example, the witness for this generator